knitr::opts_chunk$set(echo = TRUE)
options(knitr.table.format = "html") 
options(digits=5)
options(scipen = 100)
knitr::opts_chunk$set(tidy.opts=list(width.cutoff=80), tidy=TRUE)
#install.packages("pacman")
library(pacman) #for quick load/install of packages
p_load(dplyr, readr, tidyverse,reticulate, lubridate,janitor, sqldf,googlesheets4)
p_load(skimr,splitstackshape,stringr,rqdatatable)
p_load(moments)
p_load(kableExtra)
p_load(ggplot2, plotly,echarts4r,ggpubr,forcats,scales,RColorBrewer)
p_load(ggthemes)

#Advanced EDA packages 
library("Rtsne")
library("DataExplorer") #used for basic stats, qq plots and bar-plots

Attaching package: ‘DataExplorer’

The following object is masked from ‘package:rquery’:

    drop_columns
library("SmartEDA") #for more descriptive statistics 
Registered S3 method overwritten by 'GGally':
  method from   
  +.gg   ggplot2
library("dlookr")

Attaching package: ‘dlookr’

The following objects are masked from ‘package:moments’:

    kurtosis, skewness

The following object is masked from ‘package:tidyr’:

    extract

The following object is masked from ‘package:base’:

    transform
library("ggstatsplot")
You can cite this package as:
     Patil, I. (2021). Visualizations with statistical details: The 'ggstatsplot' approach.
     Journal of Open Source Software, 6(61), 3167, doi:10.21105/joss.03167

Attaching package: ‘ggstatsplot’

The following object is masked from ‘package:wrapr’:

    :=
library("flextable") #for nicer looking tables 

Attaching package: ‘flextable’

The following objects are masked from ‘package:ggpubr’:

    border, font, rotate

The following objects are masked from ‘package:plotly’:

    highlight, style

The following objects are masked from ‘package:kableExtra’:

    as_image, footnote

The following object is masked from ‘package:purrr’:

    compose
library("summarytools")

Attaching package: ‘summarytools’

The following object is masked from ‘package:wrapr’:

    view

The following object is masked from ‘package:tibble’:

    view
library("skimr")
library("performance") #imputing outliers 

Data Summaries

Read-in Google Sheet of Final Study Dataset

Final Dataset can be found here.

Relabeling regions for countries where regions were ‘NA’ and replacing null values in country column to be ‘US’

Additionally, creating new region column to differentiate CA and US countries, and finally filtering out study that had 0 conversions.

Final_CLS_2022_Study_List <- read_csv("FinalDataset_2022_Update.csv") %>%
  mutate(region = case_when(
    country %in% c('CA', 'US', 'US + CA', 'BR', 'MX', 'CL', 'SPLATAM - (AR, CL, CO, MX, PE)',
                   'LAC-Others (BO, CR, DO, EC, GT, HN, NI, PA, PR, PY, SV, UY)', 'LAC-Others (BO, CR, DO, EC, GT, HN, NI, PA, PY)') ~ 'AMER',
    TRUE ~ region),

  country = case_when(
    is.na(country) ~ 'US', 
    TRUE ~ country),
  
  #search = case_when(
    #channel == 'Search' ~ 'Search',
    #TRUE ~ 'Non-Search'),
  
   region_v2 = case_when(
    country %in% c('CA', 'US', 'US + CA') ~ 'AMER_USCA', TRUE ~ region)) %>%
  
  filter(exposed != 0) 
Rows: 513 Columns: 22── Column specification ────────────────────────────────────────────────────────────────────────────────────────
Delimiter: ","
chr  (8): quarter, region, country, pa, channel, tactic, conversion, study_name
dbl (14): study_id, year, treatment_user_count, exposed, control_user_count, scaled_control, control, scalin...
ℹ Use `spec()` to retrieve the full column specification for this data.
ℹ Specify the column types or set `show_col_types = FALSE` to quiet this message.

#TBD
#WRITING SHEETS WILL BE COMMENTED OUT UNLESS REFRESH IS NECESSARY
#write_sheet(Final_CLS_2021_Study_List, ss = #'https://docs.google.com/spreadsheets/d/146_bmm-FgOeOQub3o4WVtt-9Aw8pqYvkQnwsHmRdZDs/edit#gid#=587747443',
#          sheet = 'Final_Clean_Dataset_v2')

skim(Final_CLS_2022_Study_List) %>%
  as.tibble() %>%
  select(skim_type, skim_variable,complete_rate)
Warning: `as.tibble()` was deprecated in tibble 2.0.0.
Please use `as_tibble()` instead.
The signature and semantics have changed, see `?as_tibble`.
nrow(Final_CLS_2022_Study_List)
[1] 512

Create Group Conversion Events

Look at all current conversion events

Final_CLS_2022_Study_List %>%
  filter(channel =="Search") %>%
  group_by(pa, conversion) %>% 
  summarize(count = n()) %>% 
    kbl() %>% 
 kable_material(c("striped", "hover","condensed","responsive"),full_width = F,fixed_thead = T)
`summarise()` has grouped output by 'pa'. You can override using the `.groups` argument.
pa conversion count
Chrome Desktop Downloads 94
Chromebook Chromebook Microsite Referral Clicks Q4 2015 13
DSM Global - Google Store - All Products - Order Complete - 390-271-2647) 20
Google Cloud GWS MCC - Free Trial Submit. 35
Pixel All Mobile Pseudo Conversions 7
Pixel Google Store - Product - Pixel 6 - Order Complete - 390-271-2647) 3

##Creating Grouped conversion events


Final_CLS_2022_Study_List =
  Final_CLS_2022_Study_List %>% 
 mutate(
   parsed_type = parse_number(conversion),
   grouped_conversion = case_when(
   conversion %in% c('Chromebook Microsite Referral Clicks Q4 2015','Type 251422729 (Chromebooks Microsite Referral Clicks (Q4 2017))') ~ 'Chromebook Referrals',
   conversion %in% c('Desktop Downloads','Type 11541547 (Desktop Download)') ~      
     'Desktop Downloads',
   pa == 'Pixel'~ 'Mobile Conversions',
   pa == 'DSM' ~ 'Non-Mobile Device Conversions',
   conversion == 'Type 302982954 (Lena - P Lead)' ~ 'Lena P Lead' , 
    conversion == 'Type 288347008 (LENA - B Lead)' ~ 'Lena B Lead' ,
    conversion == 'Type 288697653 (LENA - Q Lead)' ~ 'Lena Q Lead' ,
   parsed_type == 330755641 ~ 'Microsite Conversions',
   parsed_type == 14257803 ~ 'Enterprise Signups',
   parsed_type == 289680712 ~ 'Google(iOs) First Open',
   parsed_type ==  256522942 ~ 'YouTube TV - Web - Trial Start',
  parsed_type ==  452391534 ~ 'Trial Signups Complete' ,
   TRUE ~ conversion
   )
) 

Final_CLS_2022_Study_List %>% 
  filter(channel == 'Search') %>%
  group_by(pa, grouped_conversion) %>% 
  summarize(count = n())%>% 
    kbl() %>% 
 kable_material(c("striped", "hover","condensed","responsive"),full_width = F,fixed_thead = T)
`summarise()` has grouped output by 'pa'. You can override using the `.groups` argument.
pa grouped_conversion count
Chrome Desktop Downloads 94
Chromebook Chromebook Referrals 13
DSM Non-Mobile Device Conversions 20
Google Cloud GWS MCC - Free Trial Submit. 35
Pixel Mobile Conversions 10

Checking frequencies for Search Data

Quarters in Search Data

Final_CLS_2022_Study_List %>%
filter(channel == 'Search') %>% 
  group_by(quarter) %>%
  summarize(Overall_Studies = n_distinct(study_id,quarter)) %>%
  #summarize(Overall_Studies = n()) %>%
  ggplot(aes(x=quarter, y=Overall_Studies)) + geom_bar(stat = 'identity', fill = "lightblue") +
  ggtitle("Quarters in Search Studies") +
    xlab("Quarter") +
    ylab("Count") +
  coord_flip() +
  theme_minimal()

PA’s in Search Data

Final_CLS_2022_Study_List %>%
filter(channel == 'Search') %>% 
  group_by(pa) %>%
  summarize(Overall_Studies = n_distinct(study_id,quarter)) %>%
  ggplot(aes(x=pa, y=Overall_Studies)) + geom_bar(stat = 'identity', fill = 'lightblue') +
  ggtitle("PA's in Search Studies") +
    xlab("PA") +
    ylab("Count") +
  coord_flip() +
  theme_minimal()

Regions in Search Data

Final_CLS_2022_Study_List %>%
filter(channel == 'Search') %>% 
  filter(tactic != "All") %>%
  group_by(region_v2) %>%
  #summarize(Overall_Studies = n_distinct(study_id,quarter)) %>%
  summarize(Overall_Studies = n()) %>%
  ggplot(aes(x=region_v2, y=Overall_Studies)) + geom_bar(stat = 'identity', fill = 'lightblue') +
  ggtitle("Regions in Search Studies") +
    xlab("Region") +
    ylab("Count") +
  coord_flip() +
  theme_minimal()

Tactics in Search Data

Final_CLS_2022_Study_List %>%
filter(channel == 'Search') %>% 
  filter(tactic != 'All') %>%
  group_by(tactic) %>%
  #summarize(Overall_Studies = n_distinct(study_id,quarter)) %>%
  summarize(Overall_Studies = n()) %>%
  ggplot(aes(x=tactic, y=Overall_Studies)) + geom_bar(stat = 'identity', fill = 'lightblue') +
  ggtitle("Tactics in Search Studies") +
    xlab("Tactic") +
    ylab("Count") +
  coord_flip() +
  theme_minimal()

Boxplot of Cost Spent by Grouped Conversion Event

#my_xlab <- paste(levels(Final_CLS_2021_Study_List$grouped_conversion),"\n(N=",table(Final_CLS_2021_Study_List$grouped_conversion),")",sep="")

n_fun <- function(x){
  return(data.frame(y = median(x), label = paste0("n = ",length(x))))
}


p <-
Final_CLS_2022_Study_List %>%
  filter(channel =='Search') %>% 
  filter(tactic != 'All') %>%

  mutate(class = fct_reorder(as.factor(grouped_conversion), cost_spent_on_exposed_group, .fun='median')) %>%
  ggplot() +
    aes(
      x = reorder(as.factor(class),cost_spent_on_exposed_group),y = cost_spent_on_exposed_group, fill = grouped_conversion
        #absolute_lift*100/cost_spent_on_exposed_group
        ) +
  geom_boxplot() +
stat_summary(fun.data = n_fun, geom = "text", size = 2) +
    theme_bw() +
    theme(legend.position="none",
          axis.text.x=element_blank(),
           axis.title.y = element_blank()
          ) +
    xlab("Grouped Conversion Type")+
 #    scale_x_discrete(labels=my_xlab) +
    ylab("Cost Spent on Exposed Group") +
  coord_flip()

ggplotly(p, tooltip = c("text"))

Boxplot of Absolute Lift by Grouped Conversion Event

#my_xlab <- paste(levels(Final_CLS_2021_Study_List$grouped_conversion),"\n(N=",table(Final_CLS_2021_Study_List$grouped_conversion),")",sep="")

n_fun <- function(x){
  return(data.frame(y = median(x), label = paste0("n = ",length(x))))
}


p <-
Final_CLS_2022_Study_List %>%
  filter(channel =='Search') %>% 
  filter(tactic !='All') %>%

  mutate(class = fct_reorder(as.factor(grouped_conversion), absolute_lift, .fun='median')) %>%
  ggplot() +
    aes(
      x = reorder(as.factor(class),absolute_lift),y = absolute_lift, fill = grouped_conversion
        #absolute_lift*100/cost_spent_on_exposed_group
        ) +
  geom_boxplot() +
stat_summary(fun.data = n_fun, geom = "text", size = 2) +
    theme_bw() +
    theme(legend.position="none",
          axis.text.x=element_blank(),
           axis.title.y = element_blank()
          ) +
    xlab("Grouped Conversion Type")+
 #    scale_x_discrete(labels=my_xlab) +
    ylab("Lift") +
  coord_flip()

ggplotly(p, tooltip = c("text"))

Boxplot of Absolute Lift by Grouped Conversion Event - Excluding Desktop Downloads

#my_xlab <- paste(levels(Final_CLS_2021_Study_List$grouped_conversion),"\n(N=",table(Final_CLS_2021_Study_List$grouped_conversion),")",sep="")

n_fun <- function(x){
  return(data.frame(y = median(x), label = paste0("n = ",length(x))))
}


p <-
Final_CLS_2022_Study_List %>%
  filter(channel =='Search') %>% 
  filter(conversion != 'Desktop Downloads') %>%
  filter(tactic!='All') %>% 

  mutate(class = fct_reorder(as.factor(grouped_conversion), absolute_lift, .fun='median')) %>%
  ggplot() +
    aes(
      x = reorder(as.factor(class),absolute_lift),y = absolute_lift, fill = grouped_conversion
        #absolute_lift*100/cost_spent_on_exposed_group
        ) +
  geom_boxplot() +
stat_summary(fun.data = n_fun, geom = "text", size = 2) +
    theme_bw() +
    theme(legend.position="none",
          axis.text.x=element_blank(),
           axis.title.y = element_blank()
          ) +
    xlab("Grouped Conversion Type")+
 #    scale_x_discrete(labels=my_xlab) +
    ylab("Lift") +
  coord_flip()

ggplotly(p, tooltip = c("text"))

Summary of Findings

  1. Very Significant positive lift event outliers in Free Trial Submit and Non-Mobile Device Conversions

  2. Negative lifts in Desktop Downloads, Non-Mobile Device Conversions and Mobile Conversions

  3. Much larger variation in Non-Mobile Device Conversions than Mobile Conversions

  4. Desktop Downloads has largest range in values with minimum at -3252 and maximum at 171269

  5. Desktop Downloads had maximum values of absolute lift of all conversion events, but lowest range of cost spent with the exception of outliers

  6. Non-Mobile Device Conversions cost median 3 times bigger than Mobile Conversions had much larger range on cost spent than Mobile Conversions, but lift was not significantly higher and included significantly more negative lift

##Count of negative lifts by Grouped Conversion

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  group_by(grouped_conversion) %>%
  summarise(Negative_Count = sum(absolute_lift < 0), Percent_Negative = 100 * (sum(Negative_Count)/n()))

Boxplot of Cost Spent by Region

#my_xlab <- paste(levels(Final_CLS_2021_Study_List$grouped_conversion),"\n(N=",table(Final_CLS_2021_Study_List$grouped_conversion),")",sep="")

n_fun <- function(x){
  return(data.frame(y = median(x), label = paste0("n = ",length(x))))
}


p <-
Final_CLS_2022_Study_List %>%
  filter(channel =='Search') %>% 
  filter(tactic != 'All') %>%

  mutate(class = fct_reorder(as.factor(region_v2), cost_spent_on_exposed_group, .fun='median')) %>%
  ggplot() +
    aes(
      x = reorder(as.factor(class),cost_spent_on_exposed_group),y = cost_spent_on_exposed_group, fill = region_v2
        #absolute_lift*100/cost_spent_on_exposed_group
        ) +
  geom_boxplot() +
stat_summary(fun.data = n_fun, geom = "text", size = 2) +
    theme_bw() +
    theme(legend.position="none",
          axis.text.x=element_blank(),
           axis.title.y = element_blank()
          ) +
    xlab("Region Type")+
 #    scale_x_discrete(labels=my_xlab) +
    ylab("Cost Spent on Exposed Group") +
  coord_flip()

ggplotly(p, tooltip = c("text"))

Boxplot of Absolute Lift by Region

#my_xlab <- paste(levels(Final_CLS_2021_Study_List$grouped_conversion),"\n(N=",table(Final_CLS_2021_Study_List$grouped_conversion),")",sep="")

n_fun <- function(x){
  return(data.frame(y = median(x), label = paste0("n = ",length(x))))
}


p <-
Final_CLS_2022_Study_List %>%
  filter(channel =='Search') %>% 
  filter(tactic != 'All') %>%

  mutate(class = fct_reorder(as.factor(region_v2), absolute_lift, .fun='median')) %>%
  ggplot() +
    aes(
      x = reorder(as.factor(class),absolute_lift),y = absolute_lift, fill = region_v2
        #absolute_lift*100/cost_spent_on_exposed_group
        ) +
  geom_boxplot() +
stat_summary(fun.data = n_fun, geom = "text", size = 2) +
    theme_bw() +
    theme(legend.position="none",
          axis.text.x=element_blank(),
           axis.title.y = element_blank()
          ) +
    xlab("Region Type")+
 #    scale_x_discrete(labels=my_xlab) +
    ylab("Lift") +
  coord_flip()

ggplotly(p, tooltip = c("text"))

Boxplot of Absolute Lift by Region - Excluding AO

#my_xlab <- paste(levels(Final_CLS_2021_Study_List$grouped_conversion),"\n(N=",table(Final_CLS_2021_Study_List$grouped_conversion),")",sep="")

n_fun <- function(x){
  return(data.frame(y = median(x), label = paste0("n = ",length(x))))
}


p <-
Final_CLS_2022_Study_List %>%
  filter(channel =='Search') %>% 
  filter(tactic != 'All') %>%
  filter(region_v2 != "AO") %>%

  mutate(class = fct_reorder(as.factor(region_v2), absolute_lift, .fun='median')) %>%
  ggplot() +
    aes(
      x = reorder(as.factor(class),absolute_lift),y = absolute_lift, fill = region_v2
        #absolute_lift*100/cost_spent_on_exposed_group
        ) +
  geom_boxplot() +
stat_summary(fun.data = n_fun, geom = "text", size = 2) +
    theme_bw() +
    theme(legend.position="none",
          axis.text.x=element_blank(),
           axis.title.y = element_blank()
          ) +
    xlab("Region Type")+
 #    scale_x_discrete(labels=my_xlab) +
    ylab("Lift") +
  coord_flip()

ggplotly(p, tooltip = c("text"))

Summary of Findings

  1. US + CA spent on a much larger scale than the rest of countries in AMER, yet obtained negative lift values and an overall lower range than AMER

2.US + CA cost median over ten times median of the rest of AMER countries

  1. US + CA and EMEA regions had negative lifts

##Count of negative lifts by Region

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  group_by(region_v2) %>%
  summarise(Negative_Count = sum(absolute_lift < 0),
            Percent_Negative = 100 * (sum(Negative_Count)/n()))

Summarize Spend

Search Data

Overall Spend

Final_CLS_2022_Study_List %>% 
  filter(channel == 'Search') %>% 
  filter(tactic != 'All') %>% #filtering where tactic != All to exclude overall spend for each study. can also use tactic == 'All' to confirm totals add up 
  summarise(Overall_spend = sum(cost_spent_on_exposed_group),
            Overall_Studies = n_distinct(study_id,quarter)
            ) 

Spend per Study

Study 22 had maximum spend at over 5 million

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic == "All") %>%
  select(study_id, cost_spent_on_exposed_group)

Overall Spend by PA

####Significantly more was spent on Google Cloud Maximum Spend on Google Cloud and minimum on Chromebook

Final_CLS_2022_Study_List %>% 
  filter(channel == 'Search') %>% 
  filter(tactic != 'All') %>%
  group_by(pa) %>% 
  summarise(Overall_spend = sum(cost_spent_on_exposed_group),
            Overall_Studies = n_distinct(study_id,quarter),
            Average_Spend = mean(cost_spent_on_exposed_group),
            Overall_instances = n()
            ) 

Significant Study % - Search Data

About 92% of Search spend is Significant

Final_CLS_2022_Study_List %>% 
  filter(channel == 'Search') %>% 
  filter(tactic != 'All') %>% 
  group_by(Significant_Spend) %>%
  summarise(Overall_spend = sum(cost_spent_on_exposed_group))%>% 
  mutate(Percent_spend = 100 * Overall_spend/(sum(Overall_spend)))

Significant Spend by PA

Final_CLS_2022_Study_List %>%
  filter(channel == 'Search') %>%
  filter(tactic != 'All') %>%
  group_by(pa, Significant_Spend) %>%
  summarise(Overall_spend = sum(cost_spent_on_exposed_group))%>% 
  mutate(Percent_spend = 100 * Overall_spend/(sum(Overall_spend)))
`summarise()` has grouped output by 'pa'. You can override using the `.groups` argument.

Grouped Bar Chart for Significant Spend by PA

Final_CLS_2022_Study_List %>%
  filter(channel == 'Search') %>%
  filter(tactic != 'All') %>%
  group_by(pa, Significant_Spend) %>%
  summarise(Overall_spend = sum(cost_spent_on_exposed_group))%>% 
  ggplot(aes(x=pa, y=Overall_spend, fill=as.factor(Significant_Spend))) + 
  geom_bar(position="dodge", stat="identity") +
  xlab("PA") +
  ylab("Total Spend") +
  ggtitle("Significant Spend by PA") +
  coord_flip() +
  theme_minimal()
`summarise()` has grouped output by 'pa'. You can override using the `.groups` argument.

NA

Grouped Bar Chart Significant Studies by PA - excluding PA’s that are 100% Significant - Chromebook & Google Cloud

Final_CLS_2022_Study_List %>%
 filter(channel == 'Search') %>%
 filter(tactic != 'All') %>%
 filter(!pa %in% c("Chromebook", "Google Cloud")) %>%
 group_by(pa, Significant_Spend) %>%
  summarise(Overall_spend = sum(cost_spent_on_exposed_group))%>% 
  ggplot(aes(x=pa, y=Overall_spend, fill=as.factor(Significant_Spend))) + 
  geom_bar(position="dodge", stat="identity") +
  xlab("PA") +
  ylab("Total Spend") +
  ggtitle("Spend by PA and Significant Spend Excluding Chromebook & Google Cloud") +
  coord_flip() +
  theme_minimal()
`summarise()` has grouped output by 'pa'. You can override using the `.groups` argument.


Summarize Findings

  1. All of spend on Google Cloud and Chromebook was significant

  2. Pixel was the only PA with majority of spend as non-significant, keep in mind it only appeared in 7 instances of the data

  3. Google Cloud averaged highest spend and had largest overall spend

Significant Spend by Region

Final_CLS_2022_Study_List %>%
  filter(channel == 'Search') %>%
  filter(tactic != 'All') %>%
  group_by(region_v2, Significant_Spend) %>%
  summarise(Overall_spend = sum(cost_spent_on_exposed_group))%>% 
  mutate(Percent_spend = 100 * Overall_spend/(sum(Overall_spend)))
`summarise()` has grouped output by 'region_v2'. You can override using the `.groups` argument.

Bar Chart of Significant Spend by Region

Final_CLS_2022_Study_List %>%
  filter(channel == 'Search') %>%
  filter(tactic != 'All') %>%
  group_by(region_v2, Significant_Spend) %>%
  summarise(Overall_spend = sum(cost_spent_on_exposed_group)) %>% 
  ggplot(aes(x=region_v2, y=Overall_spend, fill=as.factor(Significant_Spend))) + 
  geom_bar(position="dodge", stat="identity") +
  xlab("Region") +
  ylab("Total Spend") +
  ggtitle("Spend by Region and Significant Spend") +
  coord_flip() +
  theme_minimal()
`summarise()` has grouped output by 'region_v2'. You can override using the `.groups` argument.


Non-Significant Studies too low to be seen on barplot #### Bar Chart of Significant Spend by Region - Excluding US + CA

Final_CLS_2022_Study_List %>%
  filter(channel == 'Search') %>%
  filter(tactic != 'All') %>%
  filter(!region_v2 %in% c('AMER_USCA')) %>%
  group_by(region_v2, Significant_Spend) %>%
  summarise(Overall_spend = sum(cost_spent_on_exposed_group))%>% 
  ggplot(aes(x=region_v2, y=Overall_spend, fill=as.factor(Significant_Spend))) + 
  geom_bar(position="dodge", stat="identity") +
  xlab("Region") +
  ylab("Total Spend") +
  ggtitle("Spend by Region and Significant Spend Excluding CA & US") +
  coord_flip() +
  theme_minimal()
`summarise()` has grouped output by 'region_v2'. You can override using the `.groups` argument.

Significant Spend by Tactic

Final_CLS_2022_Study_List %>%
  filter(channel == 'Search') %>%
  filter(tactic != 'All') %>%
  group_by(tactic, Significant_Spend) %>%
  summarise(Overall_spend = sum(cost_spent_on_exposed_group))%>% 
  mutate(Percent_spend = 100 * Overall_spend/(sum(Overall_spend)))
`summarise()` has grouped output by 'tactic'. You can override using the `.groups` argument.

Bar chart of Significant Spend by Tactic

Final_CLS_2022_Study_List %>%
  filter(channel == 'Search') %>%
  filter(tactic != 'All') %>%
  group_by(tactic, Significant_Spend) %>%
  summarise(Overall_spend = sum(cost_spent_on_exposed_group))%>% 
  ggplot(aes(x=tactic, y=Overall_spend, fill=as.factor(Significant_Spend))) + 
  geom_bar(position="dodge", stat="identity") +
  xlab("Tactic") +
  ylab("Total Spend") +
  ggtitle("Significant Spend by Tactic") +
  coord_flip() +
  theme_minimal()
`summarise()` has grouped output by 'tactic'. You can override using the `.groups` argument.

NA

Summary of Findings

  1. All regions with the exception of CA + US were 99%+ Significant, CA + US were 84% significant

  2. Significant Spend % within SKWS was highest when compared to other tactics

###3D Scatterplots to see relationships between spend, duration and lift

#install.packages("scatterplot3d")
#install.packages("rgl")
#install.packages("rdlwidget")
#library(scatterplot3d)
#library("rgl")


#Scatter3D_Graph <- Final_CLS_2022_Study_List%>%
 # filter(channel == 'Search') %>%
  #filter(tactic !='All')
  
scatterplot3d(Scatter3D_Graph$cost_spent_on_exposed_group,
              Scatter3D_Graph$duration,
              Scatter3D_Graph$absolute_lift,
              main="3D Scatterplot",
              angle=55,
              xlab = "Cost Spent",
              ylab = "Duration",
              zlab = "Lift")


plot3d(Scatter3D_Graph$cost_spent_on_exposed_group,
              Scatter3D_Graph$duration,
              Scatter3D_Graph$absolute_lift,
              main="3D Scatterplot",
              angle=55,
              xlab = "Cost Spent",
              ylab = "Duration",
              zlab = "Lift")

##Scatterplot of Total Cost Spent vs. Absolute Lift for all Search

Final_CLS_2022_Study_List %>%
  filter(channel =='Search') %>%
  filter(tactic != 'All')%>%
  ggplot(aes(x = cost_spent_on_exposed_group, y = absolute_lift)) +
  geom_point() +
       xlab("Cost Spent") +
       ylab("Lift") +
       ggtitle("Cost vs. Lift")

#Scatterplot of Total Cost vs. Absolute Lift, for each individual study

Final_CLS_2022_Study_List %>%
  filter(channel =='Search') %>%
  filter(tactic == 'All')%>%
  ggplot(aes(x = cost_spent_on_exposed_group, y = absolute_lift)) +
  geom_point() +
       xlab("Cost Spent") +
       ylab("Lift") +
       ggtitle("Cost vs. Lift")

##Scatterplot of Cost Spent vs. Absolute Lift, by Region

Final_CLS_2022_Study_List %>%
  filter(channel =='Search') %>%
  filter(tactic != 'All')%>%
  ggplot(aes(x = cost_spent_on_exposed_group, y = absolute_lift, color = region_v2)) +
  geom_point() +
       xlab("Cost Spent") +
       ylab("Lift") +
       ggtitle("Cost vs. Lift")

##Scatterplot of Cost Spent vs. Absolute Lift, by Conversion Group

Final_CLS_2022_Study_List %>%
  filter(channel =='Search') %>%
  filter(tactic != 'All')%>%
  ggplot(aes(x = cost_spent_on_exposed_group, y = absolute_lift, color = grouped_conversion)) +
  geom_point() +
       xlab("Cost Spent") +
       ylab("Lift") +
       ggtitle("Cost vs. Lift")

Summary of Findings

  1. Majority of studies with low cost and high lift are Desktop Downloads

  2. US + CA tends to be high cost, low lift

  3. Desktop Downloads conversion has lowest spend with highest lift

##Scatterplot of Cost Spent vs. Absolute Lift, by Year

Generally, 2022 has higher range in spend with a large range in lift than 2021

Final_CLS_2022_Study_List %>%
  filter(channel =='Search') %>%
  filter(tactic != 'All')%>%
  ggplot(aes(x = cost_spent_on_exposed_group, y = absolute_lift, color = as.factor(year))) +
  geom_point() +
       xlab("Cost Spent") +
       ylab("Lift") +
       ggtitle("Cost vs. Lift")

##Scatterplot of Cost Spent vs Absolute Lift, by Significant Spend

Final_CLS_2022_Study_List %>%
  filter(channel =='Search') %>%
  filter(tactic != 'All')%>%
  ggplot(aes(x = cost_spent_on_exposed_group, y = absolute_lift, color = as.factor(Significant_Spend))) +
  geom_point() +
       xlab("Cost Spent") +
       ylab("Lift") +
       ggtitle("Cost vs. Lift Search")

##Scatterplot of Duration vs. Cost Spent for all Search ### Not as linear as expected

Final_CLS_2022_Study_List %>%
  filter(channel =='Search') %>%
  filter(tactic == 'All')%>%
  ggplot(aes(x = duration, y = cost_spent_on_exposed_group)) +
  geom_point() +
       xlab("Duration") +
       ylab("Cost") +
       ggtitle("Duration vs. Cost")


Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic == "All") %>%
  filter(cost_spent_on_exposed_group == max(cost_spent_on_exposed_group)) %>%
  select(study_id)

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic == "All") %>%
  filter(duration == max(duration)) %>%
  select(study_id)
NA

##Scatterplot of Duration vs. Lift, by Region

Final_CLS_2022_Study_List %>%
  filter(channel =='Search') %>%
  filter(tactic != 'All')%>%
  ggplot(aes(x = duration, y = absolute_lift, color = region_v2)) +
  geom_point() +
       xlab("Duration") +
       ylab("Lift") +
       ggtitle("Duration vs. Lift")

##Scatterplot of Duration vs. Lift, by PA

Final_CLS_2022_Study_List %>%
  filter(channel =='Search') %>%
  filter(tactic != 'All')%>%
  ggplot(aes(x = duration, y = absolute_lift, color = pa)) +
  geom_point() +
       xlab("Duration") +
       ylab("Lift") +
       ggtitle("Duration vs. Lift") 

Summary of Findings

1.Study 22 has max cost spent, yet on the lower range of duration

2.Study 1 has max duration and very low cost

Scatterplots on Individual PA’s

Scatterplots of Cost Spent vs. Lift for each PA

pa_plot <- function(pa2) {
  plot_scatter <- Final_CLS_2022_Study_List %>%
    filter(channel == 'Search') %>%
    filter(tactic != 'All') %>%
    filter(pa == pa2) %>%
    ggplot(aes(x = cost_spent_on_exposed_group, y = absolute_lift, color =region_v2)) +
    geom_point() +
       xlab("Cost") +
       ylab("Lift") +
       ggtitle(paste(pa2, " Cost vs. Lift"))
print(plot_scatter)
}

pa_columns <- c("Chrome", "Chromebook", "DSM", "Google Cloud", "Pixel")

for (i in pa_columns) {
  pa_plot(i)
} 

NA
NA

Summary of Findings

  1. Most linear trends seem to be Chromebook, DSM and Google Cloud

  2. DSM only contains AMER regions

  3. Chrome costs remain low with high lift, performed worst in US + CA region

  4. For Chrome, Chromebook and Google Cloud, it generally requires higher cost for lift in US + CA regions

Scatterplots of Cost Spent vs. Lift for each Region

region_plot <- function(region2) {
  plot_scatter <- Final_CLS_2022_Study_List %>%
    filter(channel == 'Search') %>%
    filter(tactic != 'All') %>%
    filter(region_v2 == region2) %>%
    ggplot(aes(x = cost_spent_on_exposed_group, y = absolute_lift, color = pa)) +
    geom_point() +
       xlab("Cost") +
       ylab("Lift") +
       ggtitle(paste(region2, " Cost vs. Lift"))
print(plot_scatter)
}


region_cols <- c("AMER","AMER_USCA", "AO", "APAC", "EMEA")
for (i in region_cols) {
  region_plot(i)
}


Summary of Findings

  1. EMEA region has highest lift with low costs

  2. In all regions Chrome seemed to be highest performing, with low costs resulting in high lift

  3. US + CA cost vs. lift relationship much more dispersed than rest of AMER region

  4. Google Cloud seems to require significant cost for lift across all regions when compared to all pa’s

Chrome data

Final_CLS_2022_Study_List %>%
  filter(channel =='Search') %>%
  filter(tactic != 'All')%>%
  filter(pa == 'Chrome') %>%
  summary()
    study_id         year        quarter             region            country               pa           
 Min.   : 2.0   Min.   :2021   Length:92          Length:92          Length:92          Length:92         
 1st Qu.: 2.0   1st Qu.:2021   Class :character   Class :character   Class :character   Class :character  
 Median : 9.5   Median :2022   Mode  :character   Mode  :character   Mode  :character   Mode  :character  
 Mean   : 9.5   Mean   :2022                                                                              
 3rd Qu.:17.0   3rd Qu.:2022                                                                              
 Max.   :17.0   Max.   :2022                                                                              
                                                                                                          
   channel             tactic           conversion         study_name        treatment_user_count
 Length:92          Length:92          Length:92          Length:92          Min.   :    654     
 Class :character   Class :character   Class :character   Class :character   1st Qu.:  29666     
 Mode  :character   Mode  :character   Mode  :character   Mode  :character   Median : 102492     
                                                                             Mean   : 177532     
                                                                             3rd Qu.: 185546     
                                                                             Max.   :1645490     
                                                                                                 
    exposed        control_user_count scaled_control       control      scaling_factor
 Min.   :    139   Min.   :   34      Min.   :     19   Min.   :    1   Min.   :19.2  
 1st Qu.:   5505   1st Qu.:  769      1st Qu.:   3817   1st Qu.:   88   1st Qu.:23.7  
 Median :  59904   Median : 2098      Median :  51396   Median : 1181   Median :46.4  
 Mean   : 112396   Mean   : 4380      Mean   :  99313   Mean   : 2357   Mean   :46.7  
 3rd Qu.: 111601   3rd Qu.: 4133      3rd Qu.: 100093   3rd Qu.: 2110   3rd Qu.:69.9  
 Max.   :1120981   Max.   :69027      Max.   :1068200   Max.   :38842   Max.   :76.8  
                                                                                      
 cost_spent_on_exposed_group absolute_lift    relative_lift     probability_of_lift    duration   
 Min.   :    90              Min.   : -3252   Min.   :-0.0373   Min.   :0.000       Min.   :47.0  
 1st Qu.:  2207              1st Qu.:  1306   1st Qu.: 0.0898   1st Qu.:1.000       1st Qu.:47.0  
 Median :  4121              Median :  6967   Median : 0.1603   Median :1.000       Median :67.5  
 Mean   : 19748              Mean   : 13083   Mean   : 0.4246   Mean   :0.949       Mean   :67.5  
 3rd Qu.: 12436              3rd Qu.: 13439   3rd Qu.: 0.3061   3rd Qu.:1.000       3rd Qu.:88.0  
 Max.   :625238              Max.   :171270   Max.   : 6.2263   Max.   :1.000       Max.   :88.0  
                                                                                                  
 Significant_Spend  region_v2          parsed_type  grouped_conversion
 Min.   :0.000     Length:92          Min.   : NA   Length:92         
 1st Qu.:1.000     Class :character   1st Qu.: NA   Class :character  
 Median :1.000     Mode  :character   Median : NA   Mode  :character  
 Mean   :0.924                        Mean   :NaN                     
 3rd Qu.:1.000                        3rd Qu.: NA                     
 Max.   :1.000                        Max.   : NA                     
                                      NA's   :92                      

Final_CLS_2022_Study_List %>%
  filter(channel =='Search') %>%
  filter(tactic != 'All')%>%
  filter(pa == 'Chrome') %>% 
  ggplot(aes(x = duration, y = absolute_lift)) +
  geom_point() +
       xlab("Duration") +
       ylab("Lift") +
       ggtitle("Duration vs. Lift")

Final_CLS_2022_Study_List %>%
  filter(channel =='Search') %>%
  filter(tactic != 'All')%>%
  filter(pa == 'Chrome') %>% 
  ggplot(aes(x = duration, y = cost_spent_on_exposed_group)) +
  geom_point() +
       xlab("Duration") +
       ylab("Cost") +
       ggtitle("Duration vs. Cost")

##Summary of Findings 1. Duration seems to be either short or long, and doesnt seem to have any apparent effect on cost or lift, with only a slight increase in lift for longer studies

  1. For Chrome data the duration was not a significant factor in lift

Advanced EDA

Final_CLS_2022_Study_List[c("year","quarter","region_v2","pa","tactic","Significant_Spend", "channel")] %>%
  filter(channel =="Search") %>%
  filter(tactic != "All") %>%
  plot_bar(ncol =3)

NA

Percentage of each variable by PA

Final_CLS_2022_Study_List[c("year","quarter","region_v2","pa","tactic","Significant_Spend", "channel")] %>%
  filter(channel =="Search") %>%
  filter(tactic != "All") %>%
  plot_bar(by = "pa")


##Frequency and Percentage for each variable

Final_CLS_2022_Study_List[c("year","quarter","region_v2","pa","tactic","Significant_Spend", "channel")] %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  ExpCatViz()
[[1]]

[[2]]

[[3]]

[[4]]

[[5]]

[[6]]

NA

Visualizing % of Significant Spend by Grouped Conversion, Tactic and Region using ExpCatViz

% of Significant Spend by Grouped Conversion

ExpCatViz(
  Final_CLS_2022_Study_List %>%
    filter(channel == "Search") %>%
    filter(tactic != "All") %>%
    select(grouped_conversion, Significant_Spend),
  target="grouped_conversion")
[[1]]


% Significant Spend by Tactic

ExpCatViz(
  Final_CLS_2022_Study_List %>%
    filter(channel == "Search") %>%
    filter(tactic != "All") %>%
    select(tactic, Significant_Spend),
  target="tactic")
[[1]]


% Significant Spend by Region

ExpCatViz(
  Final_CLS_2022_Study_List %>%
    filter(channel == "Search") %>%
    filter(tactic != "All") %>%
    select(region_v2, Significant_Spend),
  target="region_v2")
[[1]]


% of Significant Spend by PA with p-values

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  ggbarstats(
    x = Significant_Spend,
    y = pa,
    label = "both"
  )


% of Significant Spend by Region with p-values

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  ggbarstats(
    x = Significant_Spend,
    y = region_v2,
    label = "both"
  )


Stats table of all variables in Search by pa

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  group_by(pa) %>%
  select(treatment_user_count, exposed, control_user_count, scaled_control, control, scaling_factor,
         cost_spent_on_exposed_group, absolute_lift, relative_lift, duration, pa) %>%
  univar_numeric() %>%
  knitr::kable()
Adding missing grouping variables: `pa`
described_variables pa n na mean sd se_mean IQR skewness median
absolute_lift Chrome 92 0 13082.79993 25773.64223 2687.08795 12133.10746 4.35509 6967.41376
absolute_lift Chromebook 12 0 1659.29931 1331.56632 384.39009 1695.40317 1.16398 1302.96252
absolute_lift DSM 13 0 1085.70609 2570.08364 712.81295 1012.26341 3.15017 430.94946
absolute_lift Google Cloud 31 0 5474.26034 6577.43483 1181.34217 3345.37960 2.84179 3233.98078
absolute_lift Pixel 7 0 349.38247 390.76383 147.69485 459.67187 1.01294 210.50384
control Chrome 92 0 2356.78261 4917.49528 512.68432 2021.50000 5.31051 1181.00000
control Chromebook 12 0 673.16667 612.59609 176.84126 920.75000 0.83140 439.00000
control DSM 13 0 3038.00000 3674.96181 1019.25102 4689.00000 1.80838 1300.00000
control Google Cloud 31 0 549.87097 1072.65098 192.65380 180.00000 3.74862 180.00000
control Pixel 7 0 1134.42857 1589.06638 600.61064 1094.00000 2.12469 407.00000
control_user_count Chrome 92 0 4380.14130 8587.39093 895.29739 3364.00000 5.40806 2098.50000
control_user_count Chromebook 12 0 31645.66667 17311.64520 4997.44151 14996.25000 1.16990 26420.50000
control_user_count DSM 13 0 51433.76923 57000.44515 15809.07906 48565.00000 1.61159 31039.00000
control_user_count Google Cloud 31 0 164274.74194 289301.01980 51959.99704 103818.50000 3.10578 63524.00000
control_user_count Pixel 7 0 53427.71429 45629.44601 17246.30951 47349.00000 1.01778 37802.00000
cost_spent_on_exposed_group Chrome 92 0 19748.19848 70132.17746 7311.78494 10229.38000 7.49320 4120.81000
cost_spent_on_exposed_group Chromebook 12 0 33131.32667 27710.96816 7999.46746 40132.38750 0.85740 23577.09500
cost_spent_on_exposed_group DSM 13 0 396955.39846 718023.19817 199143.80446 304646.29000 3.23201 196424.69000
cost_spent_on_exposed_group Google Cloud 31 0 407884.64419 430782.93236 77370.89867 316813.36500 1.75654 301800.18000
cost_spent_on_exposed_group Pixel 7 0 118953.97429 207044.84254 78255.59480 47527.63000 2.54860 54964.62000
duration Chrome 92 0 67.50000 20.61233 2.14898 41.00000 0.00000 67.50000
duration Chromebook 12 0 103.00000 0.00000 0.00000 0.00000 NaN 103.00000
duration DSM 13 0 77.53846 19.91907 5.52456 31.00000 -0.50715 84.00000
duration Google Cloud 31 0 62.09677 6.96350 1.25068 8.00000 0.61929 63.00000
duration Pixel 7 0 115.42857 71.60507 27.06417 76.50000 1.20916 78.00000
exposed Chrome 92 0 112395.60870 197146.27554 20553.92006 106095.50000 3.61623 59904.00000
exposed Chromebook 12 0 3176.58333 2628.51923 758.78814 4079.00000 0.95526 2219.50000
exposed DSM 13 0 26190.46154 37652.14987 10442.82746 20191.00000 1.98762 13903.00000
exposed Google Cloud 31 0 9704.16129 11225.67516 2016.19078 5278.00000 2.66262 6298.00000
exposed Pixel 7 0 2469.00000 2160.92627 816.75336 3830.50000 0.31222 1535.00000
relative_lift Chrome 92 0 0.42464 0.82856 0.08638 0.21633 4.69173 0.16034
relative_lift Chromebook 12 0 1.98290 2.09452 0.60463 1.09926 2.15197 1.11920
relative_lift DSM 13 0 0.05204 0.04485 0.01244 0.08431 0.11077 0.06228
relative_lift Google Cloud 31 0 1.50979 0.77718 0.13959 1.17907 -0.09149 1.66433
relative_lift Pixel 7 0 0.41894 0.61341 0.23185 0.31065 2.23097 0.29295
scaled_control Chrome 92 0 99312.80876 176245.17520 18374.82972 96276.13757 3.69955 51396.46526
scaled_control Chromebook 12 0 1517.28402 1381.75419 398.87808 2088.94209 0.82432 981.72023
scaled_control DSM 13 0 25104.75545 36328.78078 10075.79091 19874.31509 1.99202 12938.17838
scaled_control Google Cloud 31 0 4229.90095 5369.19320 964.33557 2201.06534 2.51467 2257.89494
scaled_control Pixel 7 0 2119.61753 1959.42332 740.59240 3420.19317 0.36940 1138.23133
scaling_factor Chrome 92 0 46.72384 23.19328 2.41807 46.21160 0.00641 46.36129
scaling_factor Chromebook 12 0 2.25328 0.01913 0.00552 0.02610 -0.44836 2.25675
scaling_factor DSM 13 0 9.89358 7.37251 2.04477 15.32290 0.23990 6.80160
scaling_factor Google Cloud 31 0 12.14811 3.12164 0.56066 3.42465 -2.25073 12.51136
scaling_factor Pixel 7 0 2.48250 1.03674 0.39185 1.24595 -0.99408 2.86323
treatment_user_count Chrome 92 0 177532.27174 282497.97556 29452.44992 155880.50000 3.59109 102492.00000
treatment_user_count Chromebook 12 0 71247.75000 38847.67943 11214.35909 33480.25000 1.16697 59908.00000
treatment_user_count DSM 13 0 321009.00000 335306.66074 92997.33525 271980.00000 2.21367 260403.00000
treatment_user_count Google Cloud 31 0 1572608.38710 2303293.16922 413683.66531 1268209.00000 3.23493 931367.00000
treatment_user_count Pixel 7 0 120181.57143 113122.29377 42756.20815 72086.50000 1.82068 110255.00000

Stats table of all variables in Search data, includes more than above tables like min, max and # of outliers

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  group_by(pa) %>%
  select(treatment_user_count, exposed, control_user_count, scaled_control, control, scaling_factor,
         cost_spent_on_exposed_group, absolute_lift, relative_lift, duration, pa) %>%
  diagnose_numeric() %>%
  flextable()
Adding missing grouping variables: `pa`Adding missing grouping variables: `pa`Adding missing grouping variables: `pa`Adding missing grouping variables: `pa`Adding missing grouping variables: `pa`Adding missing grouping variables: `pa`Adding missing grouping variables: `pa`Adding missing grouping variables: `pa`Adding missing grouping variables: `pa`Adding missing grouping variables: `pa`

variables

min

Q1

mean

median

Q3

max

zero

minus

outlier

treatment_user_count

654.000000

29,665.7500000

177,532.271739

102,492.000000

185,546.250000

1,645,490.00000

0

0

8

treatment_user_count

24,329.000000

46,842.0000000

71,247.750000

59,908.000000

80,322.250000

149,889.00000

0

0

2

treatment_user_count

40,312.000000

74,012.0000000

321,009.000000

260,403.000000

345,992.000000

1,285,238.00000

0

0

1

treatment_user_count

37,945.000000

343,664.0000000

1,572,608.387097

931,367.000000

1,611,873.000000

11,617,859.00000

0

0

3

treatment_user_count

15,801.000000

53,611.5000000

120,181.571429

110,255.000000

125,698.000000

356,596.00000

0

0

1

exposed

139.000000

5,505.2500000

112,395.608696

59,904.000000

111,600.750000

1,120,981.00000

0

0

8

exposed

513.000000

1,097.0000000

3,176.583333

2,219.500000

5,176.000000

8,695.00000

0

0

0

exposed

2,104.000000

2,853.0000000

26,190.461538

13,903.000000

23,044.000000

111,394.00000

0

0

2

exposed

884.000000

3,595.5000000

9,704.161290

6,298.000000

8,873.500000

51,998.00000

0

0

5

exposed

255.000000

656.5000000

2,469.000000

1,535.000000

4,487.000000

5,206.00000

0

0

0

control_user_count

34.000000

768.7500000

4,380.141304

2,098.500000

4,132.750000

69,027.00000

0

0

10

control_user_count

10,771.000000

20,803.0000000

31,645.666667

26,420.500000

35,799.250000

66,860.00000

0

0

2

control_user_count

5,540.000000

11,683.0000000

51,433.769231

31,039.000000

60,248.000000

175,595.00000

0

0

2

control_user_count

2,997.000000

27,384.0000000

164,274.741935

63,524.000000

131,202.500000

1,296,187.00000

0

0

5

control_user_count

5,650.000000

26,723.5000000

53,427.714286

37,802.000000

74,072.500000

128,950.00000

0

0

0

scaled_control

19.235294

3,817.1348773

99,312.808764

51,396.465265

100,093.272447

1,068,200.36600

0

0

8

scaled_control

59.430208

446.9747827

1,517.284022

981.720227

2,535.916876

4,050.99346

0

0

0

scaled_control

1,931.504020

2,805.6606210

25,104.755448

12,938.178380

22,679.975710

108,284.86830

0

0

2

scaled_control

519.100767

1,724.3705375

4,229.900950

2,257.894937

3,925.435878

21,488.94725

0

0

4

scaled_control

92.517249

557.9569654

2,119.617532

1,138.231327

3,978.150134

4,534.35995

0

0

0

control

1.000000

88.0000000

2,356.782609

1,181.000000

2,109.500000

38,842.00000

0

0

9

control

26.000000

199.5000000

673.166667

439.000000

1,120.250000

1,807.00000

0

0

0

control

288.000000

669.0000000

3,038.000000

1,300.000000

5,358.000000

12,807.00000

0

0

1

control

41.000000

135.0000000

549.870968

180.000000

315.000000

5,518.00000

0

0

5

control

91.000000

176.5000000

1,134.428571

407.000000

1,270.500000

4,549.00000

0

0

1

scaling_factor

19.235294

23.7128804

46.723841

46.361287

69.924483

76.78602

0

0

0

scaling_factor

2.214318

2.2403883

2.253276

2.256750

2.266488

2.28578

0

0

0

scaling_factor

1.770905

3.4504836

9.893576

6.801603

18.773382

19.28194

0

0

0

scaling_factor

1.937529

11.1659266

12.148111

12.511355

14.590574

14.69924

0

0

2

scaling_factor

0.996782

1.9066551

2.482502

2.863225

3.152607

3.39898

0

0

0

cost_spent_on_exposed_group

90.000000

2,207.0325000

19,748.198478

4,120.810000

12,436.412500

625,237.86000

0

0

8

cost_spent_on_exposed_group

7,050.620000

11,406.7275000

33,131.326667

23,577.095000

51,539.115000

80,842.52000

0

0

0

cost_spent_on_exposed_group

6,318.120000

53,155.5600000

396,955.398462

196,424.690000

357,801.850000

2,712,280.15000

0

0

1

cost_spent_on_exposed_group

28,022.860000

126,877.2300000

407,884.644194

301,800.180000

443,690.595000

1,536,618.12000

0

0

4

cost_spent_on_exposed_group

3,196.860000

23,770.7150000

118,953.974286

54,964.620000

71,298.345000

584,378.22000

0

0

1

absolute_lift

-3,252.225705

1,305.7008445

13,082.799934

6,967.413760

13,438.808308

171,269.68860

0

5

7

absolute_lift

363.203392

548.2542870

1,659.299312

1,302.962516

2,243.657452

4,644.00654

0

0

0

absolute_lift

-1,286.868292

47.3393795

1,085.706091

430.949458

1,059.602792

9,336.86470

0

1

1

absolute_lift

202.241667

2,366.7577395

5,474.260340

3,233.980781

5,712.137336

30,509.05275

0

0

3

absolute_lift

-26.584701

74.5324922

349.382468

210.503836

534.204363

1,044.28443

0

2

0

relative_lift

-0.037338

0.0898097

0.424635

0.160341

0.306139

6.22630

0

5

17

relative_lift

0.489627

0.8534501

1.982900

1.119202

1.952710

7.63197

0

0

1

relative_lift

-0.011884

0.0071755

0.052035

0.062282

0.091487

0.13228

0

1

0

relative_lift

0.270826

0.8375500

1.509788

1.664331

2.016623

3.21847

0

0

0

relative_lift

-0.022054

0.0710344

0.418940

0.292950

0.381686

1.75624

0

2

1

duration

47.000000

47.0000000

67.500000

67.500000

88.000000

88.00000

0

0

0

duration

103.000000

103.0000000

103.000000

103.000000

103.000000

103.00000

0

0

0

duration

45.000000

65.0000000

77.538462

84.000000

96.000000

96.00000

0

0

0

duration

55.000000

55.0000000

62.096774

63.000000

63.000000

77.00000

0

0

2

duration

67.000000

72.5000000

115.428571

78.000000

149.000000

220.00000

0

0

0


##Summary of Findings 1. Pixel PA had lowest average lift but highest average duration

  1. Chrome PA had highest average and median lift but lowest average and median cost ## Histogram of numeric features in Search data
Final_CLS_2022_Study_List %>% 
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  select(absolute_lift, cost_spent_on_exposed_group, duration, relative_lift) %>%
  plot_histogram(ncol =2)


Quantile-Quantile plots per variable to check for normality in order to determine appropriate statistical tests

The closer to the diagonal line the closer to normal distribution

Final_CLS_2022_Study_List%>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  select(absolute_lift, cost_spent_on_exposed_group, relative_lift, duration) %>%
  plot_qq()


Quantile-Quantile plot by PA

Final_CLS_2022_Study_List%>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  select(absolute_lift, cost_spent_on_exposed_group, relative_lift, duration, pa, grouped_conversion, region_v2) %>%
  plot_qq(by = "pa")


Normality Diagnosis Plots of Absolute Lift by PA

## Diagnosis plots let you see how distributions would change with log and sqrt transformation in case normality isn't met
Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  group_by(pa) %>%
  plot_normality(absolute_lift)

Normality Diagnosis Plots of Cost Spent by PA

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  group_by(pa) %>%
  plot_normality(cost_spent_on_exposed_group)


Boxplots of absolute lift, cost spent, and relative lift across all pa’s

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  select(absolute_lift, cost_spent_on_exposed_group, relative_lift, pa, region_v2, grouped_conversion) %>%
plot_boxplot(by = "pa")


Distributions of absolute lift for each pa, including comparison between pa distributions and p-values

#install.packages("PMCMRplus")

Final_CLS_2022_Study_List %>%
  filter(channel ==  "Search") %>%
  filter(tactic != "All") %>%
  select(absolute_lift, cost_spent_on_exposed_group, pa, region_v2, grouped_conversion) %>%
  ggbetweenstats(x = pa, y = absolute_lift, type = "np")


Findings: 1. All differences in distributions of absolute lift between pa’s are significant

Distributions of absolute lift for each pa, excluding Chrome

Final_CLS_2022_Study_List %>%
  filter(channel ==  "Search") %>%
  filter(tactic != "All") %>%
  filter(pa != "Chrome") %>%
  select(absolute_lift, cost_spent_on_exposed_group, pa, region_v2, grouped_conversion) %>%
  ggbetweenstats(x = pa, y = absolute_lift, type = "np")

Distributions of absolute lift for each pa, excluding Chrome and Google Cloud

Final_CLS_2022_Study_List %>%
  filter(channel ==  "Search") %>%
  filter(tactic != "All") %>%
  filter(!pa %in% c("Chrome", "Google Cloud")) %>%
  select(absolute_lift, cost_spent_on_exposed_group, pa, region_v2, grouped_conversion) %>%
  ggbetweenstats(x = pa, y = absolute_lift, type = "np")

Distributions of absolute lift for each region, including comparison between region distributions and p-values

Final_CLS_2022_Study_List %>%
  filter(channel ==  "Search") %>%
  filter(tactic != "All") %>%
  select(absolute_lift, cost_spent_on_exposed_group, pa, region_v2, grouped_conversion) %>%
  ggbetweenstats(x = region_v2, y = absolute_lift, type = "np")


Final_CLS_2022_Study_List %>%
  filter(channel ==  "Search") %>%
  filter(tactic != "All") %>%
  select(absolute_lift, cost_spent_on_exposed_group, pa, region_v2, grouped_conversion) %>%
  ggbetweenstats(x = pa, y = cost_spent_on_exposed_group, type = "np")


Final_CLS_2022_Study_List %>%
  filter(channel ==  "Search") %>%
  filter(tactic != "All") %>%
  filter(pa != "DSM") %>%
  select(absolute_lift, cost_spent_on_exposed_group, pa, region_v2, grouped_conversion) %>%
  ggbetweenstats(x = pa, y = cost_spent_on_exposed_group, type = "np")


Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  correlate(cost_spent_on_exposed_group, absolute_lift)

correlate_func <- function(pa_val) {
  plot_corr<- Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  filter(pa == pa_val) %>%
  correlate(cost_spent_on_exposed_group, absolute_lift)
  print(plot_corr)
}

pa_vals = c("Chrome", "Chromebook", "DSM", "Pixel", "Google Cloud")

for (i in pa_vals) {
  correlate_func(i)
}

Findings 1. All pa’s except Pixel have correlation coefficients >0.5 for absolute lift and cost spent 2. Chromebook and DSM have over .90 correlation between lift and cost 3. Pixel data shows negative correlation between lift and cost - only has 7 datapoints *** ##TSNE graph by grouped conversion

Final_CLS_2022_Study_List_2 <- 
  Final_CLS_2022_Study_List %>%
  filter(channel =="Search") %>%
  filter(tactic != "All") %>%
  select(-c(study_id, year, quarter)) %>%
  mutate(ID = row_number())

#extracting categorical columns 
Final_CLS_2022_Study_List_meta <- Final_CLS_2022_Study_List_2 %>%
  select(ID, pa, tactic, grouped_conversion, region_v2)

#tSNE_fit <- Final_CLS_2022_Study_List_2 %>%
 # select(ID,treatment_user_count, exposed, control_user_count, control, scaling_factor,
  #       cost_spent_on_exposed_group, absolute_lift, relative_lift, Significant_Spend) %>%
  #column_to_rownames("ID") %>%
  #scale() %>%
  #Rtsne()

#Selecting only numerical columns and fitting the data 
tSNE_fit <- Final_CLS_2022_Study_List_2 %>%
  select(ID,treatment_user_count, exposed, cost_spent_on_exposed_group, absolute_lift, relative_lift, Significant_Spend) %>%
  column_to_rownames("ID") %>%
  scale() %>%
  Rtsne()

#turning results into dataframe 
tSNE_df <- tSNE_fit$Y %>%
  as.data.frame() %>%
  rename(tSNE1="V1", tSNE2="V2") %>%
  mutate(ID = row_number())

#joining categorical and numerical columns
tSNE_df <- tSNE_df %>%
  inner_join(Final_CLS_2022_Study_List_meta, by = "ID")

tSNE_df %>% head()

tSNE_df %>%
  ggplot(aes(x = tSNE1,
             y = tSNE2,
             color = grouped_conversion)) +
  geom_point() +
  theme(legend.position="bottom")

NA

tSNE Graph by region

Final_CLS_2022_Study_List_2 <- 
  Final_CLS_2022_Study_List %>%
  filter(channel =="Search") %>%
  filter(tactic != "All") %>%
  select(-c(study_id, year, quarter)) %>%
  mutate(ID = row_number())

Final_CLS_2022_Study_List_meta <- Final_CLS_2022_Study_List_2 %>%
  select(ID, pa, tactic, grouped_conversion, region_v2)

#tSNE_fit <- Final_CLS_2022_Study_List_2 %>%
 # select(ID,treatment_user_count, exposed, control_user_count, control, scaling_factor,
  #       cost_spent_on_exposed_group, absolute_lift, relative_lift, Significant_Spend) %>%
  #column_to_rownames("ID") %>%
  #scale() %>%
  #Rtsne()

tSNE_fit <- Final_CLS_2022_Study_List_2 %>%
  select(ID,treatment_user_count, exposed, cost_spent_on_exposed_group, absolute_lift, relative_lift, Significant_Spend) %>%
  column_to_rownames("ID") %>%
  scale() %>%
  Rtsne()

tSNE_df <- tSNE_fit$Y %>%
  as.data.frame() %>%
  rename(tSNE1="V1", tSNE2="V2") %>%
  mutate(ID = row_number())

tSNE_df <- tSNE_df %>%
  inner_join(Final_CLS_2022_Study_List_meta, by = "ID")

tSNE_df %>% head()

tSNE_df %>%
  ggplot(aes(x = tSNE1,
             y = tSNE2,
             color = pa)) +
  geom_point() +
  theme(legend.position="bottom")

##Stats table of features wit and witout Outliers - calculated using IQR method

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  diagnose_outlier() %>%
  flextable()

variables

outliers_cnt

outliers_ratio

outliers_mean

with_mean

without_mean

study_id

0

0.0000

11.65161

11.65161

year

0

0.0000

2,021.53548

2,021.53548

treatment_user_count

23

14.8387

2,237,545.47826

457,762.51613

147,648.81818

exposed

8

5.1613

665,873.00000

71,207.11613

38,844.34694

control_user_count

18

11.6129

284,461.77778

44,631.42581

13,120.86861

scaled_control

9

5.8065

544,691.29931

62,111.68960

32,363.63146

control

18

11.6129

9,691.05556

1,866.98710

839.00730

scaling_factor

0

0.0000

31.27883

31.27883

cost_spent_on_exposed_group

28

18.0645

625,606.29893

134,528.59452

26,259.49433

absolute_lift

13

8.3871

58,042.93293

9,095.42653

4,614.31679

relative_lift

8

5.1613

4.09190

0.73080

0.54788

probability_of_lift

17

10.9677

0.50921

0.94617

1.00000

duration

2

1.2903

220.00000

72.17419

70.24183

Significant_Spend

14

9.0323

0.00000

0.90968

1.00000

parsed_type

0

0.0000

4.00000

4.00000


##Plotting Variables with and without Outliers

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  dplyr::select(absolute_lift, cost_spent_on_exposed_group) %>%
  plot_outlier()

NA

Plotting variables with and without outliers - excluding chrome data

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  filter(pa != "Chrome") %>%
  dplyr::select(absolute_lift, cost_spent_on_exposed_group) %>%
  plot_outlier()


#Plotting variables with and without outliers - Chrome data

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  filter(pa == "Chrome") %>%
  dplyr::select(absolute_lift, cost_spent_on_exposed_group) %>%
  plot_outlier()


Chrome data

Chrome seems to be outperforming all other pa’s

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All")
Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic == "All") %>%
  filter(absolute_lift == max(absolute_lift)) %>%
  select(study_id, pa)
Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic == "All") %>%
  filter(cost_spent_on_exposed_group == max(cost_spent_on_exposed_group)) %>%
  select(study_id, pa)

Chrome data

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  filter(pa == "Chrome") %>%
  select(absolute_lift, cost_spent_on_exposed_group, relative_lift, pa, region_v2, grouped_conversion) %>%
plot_boxplot(by = "region_v2")


Since Chrome only contains Desktop Downloads, this could be contributing to why it has such high lift

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  filter(pa == "Chrome") %>%
  select(absolute_lift, cost_spent_on_exposed_group, relative_lift, pa, region_v2, grouped_conversion) %>%
plot_boxplot(by = "grouped_conversion")


#seeing what regions are in Chrome

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  filter(pa == "Chrome") %>%
  ggplot(aes(x = cost_spent_on_exposed_group, y = absolute_lift, color = region_v2)) +
    geom_point() +
       xlab("Cost") +
       ylab("Lift") +
       ggtitle("Cost vs. Lift")

NA

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  filter(pa == "Chrome") %>%
  group_by(region_v2) %>%
  summarise(count_regions = n())

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  filter(pa == "Chrome") %>%
  group_by(Significant_Spend) %>%
  summarise(total_count = n())
scatter_stats_pa <- function(pa_val) {
  plot <- Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  filter(pa == pa_val) %>%
  ggscatterstats(
    x = cost_spent_on_exposed_group,
    y = absolute_lift, 
    type = "np")
  print(plot)
}
for (i in pa_vals) {
  scatter_stats_pa(i)
}
Registered S3 method overwritten by 'ggside':
  method from  
  +.gg   GGally

scatter_stats_region <- function(region_val) {
  plot <- Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  filter(region_v2 == region_val) %>%
  ggscatterstats(
    x = cost_spent_on_exposed_group,
    y = absolute_lift, 
    type = "np")
  print(plot)
}
for (i in region_cols) {
  scatter_stats_region(i)
}

#Evaluating stats with and without Outliers for all PA’s

Chrome

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  filter(pa == "Chrome") %>%
  select(treatment_user_count, exposed, control_user_count, cost_spent_on_exposed_group, absolute_lift) %>%
  diagnose_outlier() %>%
  flextable()

variables

outliers_cnt

outliers_ratio

outliers_mean

with_mean

without_mean

treatment_user_count

8

8.6957

958,318

177,532.3

103,171.7

exposed

8

8.6957

665,873

112,395.6

59,683.5

control_user_count

10

10.8696

21,901

4,380.1

2,243.5

cost_spent_on_exposed_group

8

8.6957

162,164

19,748.2

6,184.8

absolute_lift

7

7.6087

85,733

13,082.8

7,099.8

Chromebook

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  filter(pa == "Chromebook") %>%
  select(treatment_user_count, exposed, control_user_count, cost_spent_on_exposed_group, absolute_lift) %>%
  diagnose_outlier() %>%
  flextable()

variables

outliers_cnt

outliers_ratio

outliers_mean

with_mean

without_mean

treatment_user_count

2

16.667

145,274

71,247.8

56,442.6

exposed

0

0.000

3,176.6

3,176.6

control_user_count

2

16.667

64,622

31,645.7

25,050.5

cost_spent_on_exposed_group

0

0.000

33,131.3

33,131.3

absolute_lift

0

0.000

1,659.3

1,659.3

DSM

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  filter(pa == "Pixel") %>%
  select(treatment_user_count, exposed, control_user_count, cost_spent_on_exposed_group, absolute_lift) %>%
  diagnose_outlier() %>%
  flextable()

variables

outliers_cnt

outliers_ratio

outliers_mean

with_mean

without_mean

treatment_user_count

1

14.286

356,596

120,181.57

80,779.17

exposed

0

0.000

2,469.00

2,469.00

control_user_count

0

0.000

53,427.71

53,427.71

cost_spent_on_exposed_group

1

14.286

584,378

118,953.97

41,383.27

absolute_lift

0

0.000

349.38

349.38

Pixel

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  filter(pa == "Chrome") %>%
  select(treatment_user_count, exposed, control_user_count, cost_spent_on_exposed_group, absolute_lift) %>%
  diagnose_outlier() %>%
  flextable()

variables

outliers_cnt

outliers_ratio

outliers_mean

with_mean

without_mean

treatment_user_count

8

8.6957

958,318

177,532.3

103,171.7

exposed

8

8.6957

665,873

112,395.6

59,683.5

control_user_count

10

10.8696

21,901

4,380.1

2,243.5

cost_spent_on_exposed_group

8

8.6957

162,164

19,748.2

6,184.8

absolute_lift

7

7.6087

85,733

13,082.8

7,099.8

Google Cloud

Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  filter(pa == "Google Cloud") %>%
  select(treatment_user_count, exposed, control_user_count, cost_spent_on_exposed_group, absolute_lift) %>%
  diagnose_outlier() %>%
  flextable()

variables

outliers_cnt

outliers_ratio

outliers_mean

with_mean

without_mean

treatment_user_count

3

9.6774

7,406,089

1,572,608.4

947,592.6

exposed

5

16.1290

31,169

9,704.2

5,576.3

control_user_count

5

16.1290

690,457

164,274.7

63,085.8

cost_spent_on_exposed_group

4

12.9032

1,422,370

407,884.6

257,590.5

absolute_lift

3

9.6774

22,877

5,474.3

3,609.7

#install.packages("gtsummary")
Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  select(region_v2, pa, channel, tactic, grouped_conversion, cost_spent_on_exposed_group, absolute_lift) %>%
  tbl_summary(by = pa) %>%
  add_p()
There was an error in 'add_p()/add_difference()' for variable 'region_v2', p-value omitted:
Error in stats::fisher.test(c("AMER_USCA", "AMER_USCA", "AMER", "AMER", : FEXACT error 6 (f5xact).  LDKEY=621 is too small for this problem: kval=14082182.
Try increasing the size of the workspace.
There was an error in 'add_p()/add_difference()' for variable 'channel', p-value omitted:
Error in stats::chisq.test(x = c("Search", "Search", "Search", "Search", : 'x' and 'y' must have at least 2 levels
There was an error in 'add_p()/add_difference()' for variable 'tactic', p-value omitted:
Error in stats::fisher.test(c("AO", "SKWS", "AO", "AO", "AO", "AO", "AO", : FEXACT error 7(location). LDSTP=18630 is too small for this problem,
  (pastp=23.8233, ipn_0:=ipoin[itp=484]=5476, stp[ipn_0]=14.5866).
Increase workspace or consider using 'simulate.p.value=TRUE'
There was an error in 'add_p()/add_difference()' for variable 'grouped_conversion', p-value omitted:
Error in stats::fisher.test(c("Mobile Conversions", "Mobile Conversions", : FEXACT error 6.  LDKEY=621 is too small for this problem,
  (ii := key2[itp=602] = 4220911, ldstp=18630)
Try increasing the size of the workspace and possibly 'mult'
Characteristic Chrome, N = 921 Chromebook, N = 121 DSM, N = 131 Google Cloud, N = 311 Pixel, N = 71 p-value2
region_v2
AMER 20 (22%) 0 (0%) 4 (31%) 14 (45%) 0 (0%)
AMER_USCA 7 (7.6%) 2 (17%) 9 (69%) 3 (9.7%) 4 (57%)
AO 5 (5.4%) 2 (17%) 0 (0%) 0 (0%) 0 (0%)
APAC 35 (38%) 2 (17%) 0 (0%) 12 (39%) 0 (0%)
EMEA 25 (27%) 6 (50%) 0 (0%) 2 (6.5%) 3 (43%)
channel
Search 92 (100%) 12 (100%) 13 (100%) 31 (100%) 7 (100%)
tactic
AO 23 (25%) 0 (0%) 11 (85%) 10 (32%) 5 (71%)
BKWS 32 (35%) 6 (50%) 2 (15%) 11 (35%) 1 (14%)
SKWS 37 (40%) 6 (50%) 0 (0%) 10 (32%) 1 (14%)
grouped_conversion
Chromebook Referrals 0 (0%) 12 (100%) 0 (0%) 0 (0%) 0 (0%)
Desktop Downloads 92 (100%) 0 (0%) 0 (0%) 0 (0%) 0 (0%)
GWS MCC - Free Trial Submit. 0 (0%) 0 (0%) 0 (0%) 31 (100%) 0 (0%)
Mobile Conversions 0 (0%) 0 (0%) 0 (0%) 0 (0%) 7 (100%)
Non-Mobile Device Conversions 0 (0%) 0 (0%) 13 (100%) 0 (0%) 0 (0%)
cost_spent_on_exposed_group 4,121 (2,207, 12,436) 23,577 (11,407, 51,539) 196,425 (53,156, 357,802) 301,800 (126,877, 443,691) 54,965 (23,771, 71,298) <0.001
absolute_lift 6,967 (1,306, 13,439) 1,303 (548, 2,244) 431 (47, 1,060) 3,234 (2,367, 5,712) 211 (75, 534) <0.001
1 n (%); Median (IQR)
2 Kruskal-Wallis rank sum test

Search_no_tactic <- Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All") %>%
  kruskal.test(absolute_lift ~ pa)
Warning: 'x' is a list, so ignoring argument 'g'Warning: some elements of 'x' are not numeric and will be coerced to numeric
Search_no_tactic <- Final_CLS_2022_Study_List %>%
  filter(channel == "Search") %>%
  filter(tactic != "All")
kruskal.test(absolute_lift ~ pa, data = Search_no_tactic)

    Kruskal-Wallis rank sum test

data:  absolute_lift by pa
Kruskal-Wallis chi-squared = 33.3, df = 4, p-value = 0.000001
kruskal.test(absolute_lift ~ region_v2, data = Search_no_tactic)

    Kruskal-Wallis rank sum test

data:  absolute_lift by region_v2
Kruskal-Wallis chi-squared = 18.4, df = 4, p-value = 0.001
pairwise.wilcox.test(Search_no_tactic$absolute_lift, Search_no_tactic$pa,
                 p.adjust.method = "BH")

    Pairwise comparisons using Wilcoxon rank sum test with continuity correction 

data:  Search_no_tactic$absolute_lift and Search_no_tactic$pa 

             Chrome Chromebook DSM    Google Cloud
Chromebook   0.009  -          -      -           
DSM          0.0009 0.032      -      -           
Google Cloud 0.147  0.004      0.0002 -           
Pixel        0.002  0.009      0.393  0.0001      

P value adjustment method: BH 
LS0tDQp0aXRsZTogIjIwMjIgQ0xTIERhdGEgU3VtbWFyaXphdGlvbiINCmF1dGhvcjogIkVzc2VuY2UgR2xvYmFsIEFkdmFuY2VkIEFuYWx5dGljcyBUZWFtIg0KZGF0ZTogImByIFN5cy5EYXRlKClgIg0Kb3V0cHV0Og0KICBodG1sX25vdGVib29rOg0KICAgIHRvYzogeWVzDQogICAgdG9jX2Zsb2F0OiB5ZXMNCiAgICBudW1iZXJfc2VjdGlvbnM6IG5vDQogICAgdGhlbWU6IGNlcnVsZWFuDQogICAgaGlnaGxpZ2h0OiB6ZW5idXJuDQogICAgZmlnX2NhcHRpb246IHllcw0KICAgIGRmX3ByaW50OiBwYWdlZA0KDQotLS0NCg0KYGBge3Igc2V0dXAsIGluY2x1ZGU9VFJVRX0NCmtuaXRyOjpvcHRzX2NodW5rJHNldChlY2hvID0gVFJVRSkNCm9wdGlvbnMoa25pdHIudGFibGUuZm9ybWF0ID0gImh0bWwiKSANCm9wdGlvbnMoZGlnaXRzPTUpDQpvcHRpb25zKHNjaXBlbiA9IDEwMCkNCmtuaXRyOjpvcHRzX2NodW5rJHNldCh0aWR5Lm9wdHM9bGlzdCh3aWR0aC5jdXRvZmY9ODApLCB0aWR5PVRSVUUpDQojaW5zdGFsbC5wYWNrYWdlcygicGFjbWFuIikNCmxpYnJhcnkocGFjbWFuKSAjZm9yIHF1aWNrIGxvYWQvaW5zdGFsbCBvZiBwYWNrYWdlcw0KcF9sb2FkKGRwbHlyLCByZWFkciwgdGlkeXZlcnNlLHJldGljdWxhdGUsIGx1YnJpZGF0ZSxqYW5pdG9yLCBzcWxkZixnb29nbGVzaGVldHM0KQ0KcF9sb2FkKHNraW1yLHNwbGl0c3RhY2tzaGFwZSxzdHJpbmdyLHJxZGF0YXRhYmxlKQ0KcF9sb2FkKG1vbWVudHMpDQpwX2xvYWQoa2FibGVFeHRyYSkNCnBfbG9hZChnZ3Bsb3QyLCBwbG90bHksZWNoYXJ0czRyLGdncHVicixmb3JjYXRzLHNjYWxlcyxSQ29sb3JCcmV3ZXIpDQpwX2xvYWQoZ2d0aGVtZXMpDQoNCiNBZHZhbmNlZCBFREEgcGFja2FnZXMgDQpsaWJyYXJ5KCJSdHNuZSIpDQpsaWJyYXJ5KCJEYXRhRXhwbG9yZXIiKSAjdXNlZCBmb3IgYmFzaWMgc3RhdHMsIHFxIHBsb3RzIGFuZCBiYXItcGxvdHMNCmxpYnJhcnkoIlNtYXJ0RURBIikgI2ZvciBtb3JlIGRlc2NyaXB0aXZlIHN0YXRpc3RpY3MgDQpsaWJyYXJ5KCJkbG9va3IiKQ0KbGlicmFyeSgiZ2dzdGF0c3Bsb3QiKQ0KbGlicmFyeSgiZmxleHRhYmxlIikgI2ZvciBuaWNlciBsb29raW5nIHRhYmxlcyANCmxpYnJhcnkoInN1bW1hcnl0b29scyIpDQpsaWJyYXJ5KCJza2ltciIpDQpsaWJyYXJ5KCJwZXJmb3JtYW5jZSIpICNpbXB1dGluZyBvdXRsaWVycyANCmBgYA0KDQoqKioNCiMgRGF0YSBTdW1tYXJpZXMNCg0KIyMgKlJlYWQtaW4gR29vZ2xlIFNoZWV0IG9mIEZpbmFsIFN0dWR5IERhdGFzZXQqICANCg0KW0ZpbmFsIERhdGFzZXQgY2FuIGJlIGZvdW5kIGhlcmUuXShodHRwczovL2RvY3MuZ29vZ2xlLmNvbS9zcHJlYWRzaGVldHMvZC8xTjQ4clRlcTdtZDB2OHc4cEdfOFhJaXVhcFBIUUFlTzVXb1dJQjNlYWNlSS9lZGl0I2dpZD0xNDQ5MzUxMzc3MykgDQoNClJlbGFiZWxpbmcgcmVnaW9ucyBmb3IgY291bnRyaWVzIHdoZXJlIHJlZ2lvbnMgd2VyZSAnTkEnIGFuZCByZXBsYWNpbmcgbnVsbCB2YWx1ZXMgaW4gY291bnRyeSBjb2x1bW4gdG8gYmUgJ1VTJw0KDQpBZGRpdGlvbmFsbHksIGNyZWF0aW5nIG5ldyByZWdpb24gY29sdW1uIHRvIGRpZmZlcmVudGlhdGUgQ0EgYW5kIFVTIGNvdW50cmllcywgYW5kIGZpbmFsbHkgZmlsdGVyaW5nIG91dCBzdHVkeSB0aGF0IGhhZCAwIGNvbnZlcnNpb25zLiANCg0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0IDwtIHJlYWRfY3N2KCJGaW5hbERhdGFzZXRfMjAyMl9VcGRhdGUuY3N2IikgJT4lDQogIG11dGF0ZShyZWdpb24gPSBjYXNlX3doZW4oDQogICAgY291bnRyeSAlaW4lIGMoJ0NBJywgJ1VTJywgJ1VTICsgQ0EnLCAnQlInLCAnTVgnLCAnQ0wnLCAnU1BMQVRBTSAtIChBUiwgQ0wsIENPLCBNWCwgUEUpJywNCiAgICAgICAgICAgICAgICAgICAnTEFDLU90aGVycyAoQk8sIENSLCBETywgRUMsIEdULCBITiwgTkksIFBBLCBQUiwgUFksIFNWLCBVWSknLCAnTEFDLU90aGVycyAoQk8sIENSLCBETywgRUMsIEdULCBITiwgTkksIFBBLCBQWSknKSB+ICdBTUVSJywNCiAgICBUUlVFIH4gcmVnaW9uKSwNCg0KICBjb3VudHJ5ID0gY2FzZV93aGVuKA0KICAgIGlzLm5hKGNvdW50cnkpIH4gJ1VTJywgDQogICAgVFJVRSB+IGNvdW50cnkpLA0KICANCiAgI3NlYXJjaCA9IGNhc2Vfd2hlbigNCiAgICAjY2hhbm5lbCA9PSAnU2VhcmNoJyB+ICdTZWFyY2gnLA0KICAgICNUUlVFIH4gJ05vbi1TZWFyY2gnKSwNCiAgDQogICByZWdpb25fdjIgPSBjYXNlX3doZW4oDQogICAgY291bnRyeSAlaW4lIGMoJ0NBJywgJ1VTJywgJ1VTICsgQ0EnKSB+ICdBTUVSX1VTQ0EnLCBUUlVFIH4gcmVnaW9uKSkgJT4lDQogIA0KICBmaWx0ZXIoZXhwb3NlZCAhPSAwKSANCmBgYA0KKioqDQoNCmBgYHtyfQ0KI1RCRA0KI1dSSVRJTkcgU0hFRVRTIFdJTEwgQkUgQ09NTUVOVEVEIE9VVCBVTkxFU1MgUkVGUkVTSCBJUyBORUNFU1NBUlkNCiN3cml0ZV9zaGVldChGaW5hbF9DTFNfMjAyMV9TdHVkeV9MaXN0LCBzcyA9ICMnaHR0cHM6Ly9kb2NzLmdvb2dsZS5jb20vc3ByZWFkc2hlZXRzL2QvMTQ2X2JtbS1GZ09lT1F1YjNvNFdWdHQtOUF3OHBxWXZrUW53c0htUmRaRHMvZWRpdCNnaWQjPTU4Nzc0NzQ0MycsDQojICAgICAgICAgIHNoZWV0ID0gJ0ZpbmFsX0NsZWFuX0RhdGFzZXRfdjInKQ0KDQpza2ltKEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QpICU+JQ0KICBhcy50aWJibGUoKSAlPiUNCiAgc2VsZWN0KHNraW1fdHlwZSwgc2tpbV92YXJpYWJsZSxjb21wbGV0ZV9yYXRlKQ0KDQpucm93KEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QpDQoNCmBgYA0KKioqDQoNCiMjIENyZWF0ZSBHcm91cCBDb252ZXJzaW9uIEV2ZW50cw0KDQojIyMgTG9vayBhdCBhbGwgY3VycmVudCBjb252ZXJzaW9uIGV2ZW50cw0KDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09IlNlYXJjaCIpICU+JQ0KICBncm91cF9ieShwYSwgY29udmVyc2lvbikgJT4lIA0KICBzdW1tYXJpemUoY291bnQgPSBuKCkpICU+JSANCiAgICBrYmwoKSAlPiUgDQoga2FibGVfbWF0ZXJpYWwoYygic3RyaXBlZCIsICJob3ZlciIsImNvbmRlbnNlZCIsInJlc3BvbnNpdmUiKSxmdWxsX3dpZHRoID0gRixmaXhlZF90aGVhZCA9IFQpDQpgYGANCioqKg0KIyNDcmVhdGluZyBHcm91cGVkIGNvbnZlcnNpb24gZXZlbnRzDQpgYGB7ciwgd2FybmluZz1GQUxTRX0NCg0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCA9DQogIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lIA0KIG11dGF0ZSgNCiAgIHBhcnNlZF90eXBlID0gcGFyc2VfbnVtYmVyKGNvbnZlcnNpb24pLA0KICAgZ3JvdXBlZF9jb252ZXJzaW9uID0gY2FzZV93aGVuKA0KICAgY29udmVyc2lvbiAlaW4lIGMoJ0Nocm9tZWJvb2sgTWljcm9zaXRlIFJlZmVycmFsIENsaWNrcyBRNCAyMDE1JywnVHlwZSAyNTE0MjI3MjkgKENocm9tZWJvb2tzIE1pY3Jvc2l0ZSBSZWZlcnJhbCBDbGlja3MgKFE0IDIwMTcpKScpIH4gJ0Nocm9tZWJvb2sgUmVmZXJyYWxzJywNCiAgIGNvbnZlcnNpb24gJWluJSBjKCdEZXNrdG9wIERvd25sb2FkcycsJ1R5cGUgMTE1NDE1NDcgKERlc2t0b3AgRG93bmxvYWQpJykgfiAgICAgIA0KICAgICAnRGVza3RvcCBEb3dubG9hZHMnLA0KICAgcGEgPT0gJ1BpeGVsJ34gJ01vYmlsZSBDb252ZXJzaW9ucycsDQogICBwYSA9PSAnRFNNJyB+ICdOb24tTW9iaWxlIERldmljZSBDb252ZXJzaW9ucycsDQogICBjb252ZXJzaW9uID09ICdUeXBlIDMwMjk4Mjk1NCAoTGVuYSAtIFAgTGVhZCknIH4gJ0xlbmEgUCBMZWFkJyAsIA0KICAgIGNvbnZlcnNpb24gPT0gJ1R5cGUgMjg4MzQ3MDA4IChMRU5BIC0gQiBMZWFkKScgfiAnTGVuYSBCIExlYWQnICwNCiAgICBjb252ZXJzaW9uID09ICdUeXBlIDI4ODY5NzY1MyAoTEVOQSAtIFEgTGVhZCknIH4gJ0xlbmEgUSBMZWFkJyAsDQogICBwYXJzZWRfdHlwZSA9PSAzMzA3NTU2NDEgfiAnTWljcm9zaXRlIENvbnZlcnNpb25zJywNCiAgIHBhcnNlZF90eXBlID09IDE0MjU3ODAzIH4gJ0VudGVycHJpc2UgU2lnbnVwcycsDQogICBwYXJzZWRfdHlwZSA9PSAyODk2ODA3MTIgfiAnR29vZ2xlKGlPcykgRmlyc3QgT3BlbicsDQogICBwYXJzZWRfdHlwZSA9PSAgMjU2NTIyOTQyIH4gJ1lvdVR1YmUgVFYgLSBXZWIgLSBUcmlhbCBTdGFydCcsDQogIHBhcnNlZF90eXBlID09ICA0NTIzOTE1MzQgfiAnVHJpYWwgU2lnbnVwcyBDb21wbGV0ZScgLA0KICAgVFJVRSB+IGNvbnZlcnNpb24NCiAgICkNCikgDQoNCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lIA0KICBmaWx0ZXIoY2hhbm5lbCA9PSAnU2VhcmNoJykgJT4lDQogIGdyb3VwX2J5KHBhLCBncm91cGVkX2NvbnZlcnNpb24pICU+JSANCiAgc3VtbWFyaXplKGNvdW50ID0gbigpKSU+JSANCiAgICBrYmwoKSAlPiUgDQoga2FibGVfbWF0ZXJpYWwoYygic3RyaXBlZCIsICJob3ZlciIsImNvbmRlbnNlZCIsInJlc3BvbnNpdmUiKSxmdWxsX3dpZHRoID0gRixmaXhlZF90aGVhZCA9IFQpDQpgYGANCg0KKioqDQoNCiMjIENoZWNraW5nIGZyZXF1ZW5jaWVzIGZvciBTZWFyY2ggRGF0YSANCg0KIyMjIyBRdWFydGVycyBpbiBTZWFyY2ggRGF0YQ0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KZmlsdGVyKGNoYW5uZWwgPT0gJ1NlYXJjaCcpICU+JSANCiAgZ3JvdXBfYnkocXVhcnRlcikgJT4lDQogIHN1bW1hcml6ZShPdmVyYWxsX1N0dWRpZXMgPSBuX2Rpc3RpbmN0KHN0dWR5X2lkLHF1YXJ0ZXIpKSAlPiUNCiAgI3N1bW1hcml6ZShPdmVyYWxsX1N0dWRpZXMgPSBuKCkpICU+JQ0KICBnZ3Bsb3QoYWVzKHg9cXVhcnRlciwgeT1PdmVyYWxsX1N0dWRpZXMpKSArIGdlb21fYmFyKHN0YXQgPSAnaWRlbnRpdHknLCBmaWxsID0gImxpZ2h0Ymx1ZSIpICsNCiAgZ2d0aXRsZSgiUXVhcnRlcnMgaW4gU2VhcmNoIFN0dWRpZXMiKSArDQogICAgeGxhYigiUXVhcnRlciIpICsNCiAgICB5bGFiKCJDb3VudCIpICsNCiAgY29vcmRfZmxpcCgpICsNCiAgdGhlbWVfbWluaW1hbCgpDQpgYGANCg0KIyMjIyBQQSdzIGluIFNlYXJjaCBEYXRhDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQpmaWx0ZXIoY2hhbm5lbCA9PSAnU2VhcmNoJykgJT4lIA0KICBncm91cF9ieShwYSkgJT4lDQogIHN1bW1hcml6ZShPdmVyYWxsX1N0dWRpZXMgPSBuX2Rpc3RpbmN0KHN0dWR5X2lkLHF1YXJ0ZXIpKSAlPiUNCiAgZ2dwbG90KGFlcyh4PXBhLCB5PU92ZXJhbGxfU3R1ZGllcykpICsgZ2VvbV9iYXIoc3RhdCA9ICdpZGVudGl0eScsIGZpbGwgPSAnbGlnaHRibHVlJykgKw0KICBnZ3RpdGxlKCJQQSdzIGluIFNlYXJjaCBTdHVkaWVzIikgKw0KICAgIHhsYWIoIlBBIikgKw0KICAgIHlsYWIoIkNvdW50IikgKw0KICBjb29yZF9mbGlwKCkgKw0KICB0aGVtZV9taW5pbWFsKCkNCmBgYA0KDQojIyMjIFJlZ2lvbnMgaW4gU2VhcmNoIERhdGEgDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQpmaWx0ZXIoY2hhbm5lbCA9PSAnU2VhcmNoJykgJT4lIA0KICBmaWx0ZXIodGFjdGljICE9ICJBbGwiKSAlPiUNCiAgZ3JvdXBfYnkocmVnaW9uX3YyKSAlPiUNCiAgI3N1bW1hcml6ZShPdmVyYWxsX1N0dWRpZXMgPSBuX2Rpc3RpbmN0KHN0dWR5X2lkLHF1YXJ0ZXIpKSAlPiUNCiAgc3VtbWFyaXplKE92ZXJhbGxfU3R1ZGllcyA9IG4oKSkgJT4lDQogIGdncGxvdChhZXMoeD1yZWdpb25fdjIsIHk9T3ZlcmFsbF9TdHVkaWVzKSkgKyBnZW9tX2JhcihzdGF0ID0gJ2lkZW50aXR5JywgZmlsbCA9ICdsaWdodGJsdWUnKSArDQogIGdndGl0bGUoIlJlZ2lvbnMgaW4gU2VhcmNoIFN0dWRpZXMiKSArDQogICAgeGxhYigiUmVnaW9uIikgKw0KICAgIHlsYWIoIkNvdW50IikgKw0KICBjb29yZF9mbGlwKCkgKw0KICB0aGVtZV9taW5pbWFsKCkNCmBgYA0KDQojIyMjIFRhY3RpY3MgaW4gU2VhcmNoIERhdGENCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCmZpbHRlcihjaGFubmVsID09ICdTZWFyY2gnKSAlPiUgDQogIGZpbHRlcih0YWN0aWMgIT0gJ0FsbCcpICU+JQ0KICBncm91cF9ieSh0YWN0aWMpICU+JQ0KICAjc3VtbWFyaXplKE92ZXJhbGxfU3R1ZGllcyA9IG5fZGlzdGluY3Qoc3R1ZHlfaWQscXVhcnRlcikpICU+JQ0KICBzdW1tYXJpemUoT3ZlcmFsbF9TdHVkaWVzID0gbigpKSAlPiUNCiAgZ2dwbG90KGFlcyh4PXRhY3RpYywgeT1PdmVyYWxsX1N0dWRpZXMpKSArIGdlb21fYmFyKHN0YXQgPSAnaWRlbnRpdHknLCBmaWxsID0gJ2xpZ2h0Ymx1ZScpICsNCiAgZ2d0aXRsZSgiVGFjdGljcyBpbiBTZWFyY2ggU3R1ZGllcyIpICsNCiAgICB4bGFiKCJUYWN0aWMiKSArDQogICAgeWxhYigiQ291bnQiKSArDQogIGNvb3JkX2ZsaXAoKSArDQogIHRoZW1lX21pbmltYWwoKQ0KYGBgDQoNCiMjIyBCb3hwbG90IG9mIENvc3QgU3BlbnQgYnkgR3JvdXBlZCBDb252ZXJzaW9uIEV2ZW50ICANCmBgYHtyIGZpZy5oZWlnaHQgPSAxMCwgZmlnLndpZHRoID0gMTB9DQojbXlfeGxhYiA8LSBwYXN0ZShsZXZlbHMoRmluYWxfQ0xTXzIwMjFfU3R1ZHlfTGlzdCRncm91cGVkX2NvbnZlcnNpb24pLCJcbihOPSIsdGFibGUoRmluYWxfQ0xTXzIwMjFfU3R1ZHlfTGlzdCRncm91cGVkX2NvbnZlcnNpb24pLCIpIixzZXA9IiIpDQoNCm5fZnVuIDwtIGZ1bmN0aW9uKHgpew0KICByZXR1cm4oZGF0YS5mcmFtZSh5ID0gbWVkaWFuKHgpLCBsYWJlbCA9IHBhc3RlMCgibiA9ICIsbGVuZ3RoKHgpKSkpDQp9DQoNCg0KcCA8LQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0nU2VhcmNoJykgJT4lIA0KICBmaWx0ZXIodGFjdGljICE9ICdBbGwnKSAlPiUNCg0KICBtdXRhdGUoY2xhc3MgPSBmY3RfcmVvcmRlcihhcy5mYWN0b3IoZ3JvdXBlZF9jb252ZXJzaW9uKSwgY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCAuZnVuPSdtZWRpYW4nKSkgJT4lDQogIGdncGxvdCgpICsNCiAgICBhZXMoDQogICAgICB4ID0gcmVvcmRlcihhcy5mYWN0b3IoY2xhc3MpLGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCkseSA9IGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCwgZmlsbCA9IGdyb3VwZWRfY29udmVyc2lvbg0KICAgICAgICAjYWJzb2x1dGVfbGlmdCoxMDAvY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwDQogICAgICAgICkgKw0KICBnZW9tX2JveHBsb3QoKSArDQpzdGF0X3N1bW1hcnkoZnVuLmRhdGEgPSBuX2Z1biwgZ2VvbSA9ICJ0ZXh0Iiwgc2l6ZSA9IDIpICsNCiAgICB0aGVtZV9idygpICsNCiAgICB0aGVtZShsZWdlbmQucG9zaXRpb249Im5vbmUiLA0KICAgICAgICAgIGF4aXMudGV4dC54PWVsZW1lbnRfYmxhbmsoKSwNCiAgICAgICAgICAgYXhpcy50aXRsZS55ID0gZWxlbWVudF9ibGFuaygpDQogICAgICAgICAgKSArDQogICAgeGxhYigiR3JvdXBlZCBDb252ZXJzaW9uIFR5cGUiKSsNCiAjICAgIHNjYWxlX3hfZGlzY3JldGUobGFiZWxzPW15X3hsYWIpICsNCiAgICB5bGFiKCJDb3N0IFNwZW50IG9uIEV4cG9zZWQgR3JvdXAiKSArDQogIGNvb3JkX2ZsaXAoKQ0KDQpnZ3Bsb3RseShwLCB0b29sdGlwID0gYygidGV4dCIpKQ0KYGBgDQoqKioNCiMjIEJveHBsb3Qgb2YgQWJzb2x1dGUgTGlmdCBieSBHcm91cGVkIENvbnZlcnNpb24gRXZlbnQgDQpgYGB7ciBmaWcuaGVpZ2h0ID0gMTAsIGZpZy53aWR0aCA9IDEwfQ0KI215X3hsYWIgPC0gcGFzdGUobGV2ZWxzKEZpbmFsX0NMU18yMDIxX1N0dWR5X0xpc3QkZ3JvdXBlZF9jb252ZXJzaW9uKSwiXG4oTj0iLHRhYmxlKEZpbmFsX0NMU18yMDIxX1N0dWR5X0xpc3QkZ3JvdXBlZF9jb252ZXJzaW9uKSwiKSIsc2VwPSIiKQ0KDQpuX2Z1biA8LSBmdW5jdGlvbih4KXsNCiAgcmV0dXJuKGRhdGEuZnJhbWUoeSA9IG1lZGlhbih4KSwgbGFiZWwgPSBwYXN0ZTAoIm4gPSAiLGxlbmd0aCh4KSkpKQ0KfQ0KDQoNCnAgPC0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09J1NlYXJjaCcpICU+JSANCiAgZmlsdGVyKHRhY3RpYyAhPSdBbGwnKSAlPiUNCg0KICBtdXRhdGUoY2xhc3MgPSBmY3RfcmVvcmRlcihhcy5mYWN0b3IoZ3JvdXBlZF9jb252ZXJzaW9uKSwgYWJzb2x1dGVfbGlmdCwgLmZ1bj0nbWVkaWFuJykpICU+JQ0KICBnZ3Bsb3QoKSArDQogICAgYWVzKA0KICAgICAgeCA9IHJlb3JkZXIoYXMuZmFjdG9yKGNsYXNzKSxhYnNvbHV0ZV9saWZ0KSx5ID0gYWJzb2x1dGVfbGlmdCwgZmlsbCA9IGdyb3VwZWRfY29udmVyc2lvbg0KICAgICAgICAjYWJzb2x1dGVfbGlmdCoxMDAvY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwDQogICAgICAgICkgKw0KICBnZW9tX2JveHBsb3QoKSArDQpzdGF0X3N1bW1hcnkoZnVuLmRhdGEgPSBuX2Z1biwgZ2VvbSA9ICJ0ZXh0Iiwgc2l6ZSA9IDIpICsNCiAgICB0aGVtZV9idygpICsNCiAgICB0aGVtZShsZWdlbmQucG9zaXRpb249Im5vbmUiLA0KICAgICAgICAgIGF4aXMudGV4dC54PWVsZW1lbnRfYmxhbmsoKSwNCiAgICAgICAgICAgYXhpcy50aXRsZS55ID0gZWxlbWVudF9ibGFuaygpDQogICAgICAgICAgKSArDQogICAgeGxhYigiR3JvdXBlZCBDb252ZXJzaW9uIFR5cGUiKSsNCiAjICAgIHNjYWxlX3hfZGlzY3JldGUobGFiZWxzPW15X3hsYWIpICsNCiAgICB5bGFiKCJMaWZ0IikgKw0KICBjb29yZF9mbGlwKCkNCg0KZ2dwbG90bHkocCwgdG9vbHRpcCA9IGMoInRleHQiKSkNCmBgYA0KDQojIyBCb3hwbG90IG9mIEFic29sdXRlIExpZnQgYnkgR3JvdXBlZCBDb252ZXJzaW9uIEV2ZW50IC0gRXhjbHVkaW5nIERlc2t0b3AgRG93bmxvYWRzIA0KYGBge3IgZmlnLmhlaWdodCA9IDEwLCBmaWcud2lkdGggPSAxMH0NCiNteV94bGFiIDwtIHBhc3RlKGxldmVscyhGaW5hbF9DTFNfMjAyMV9TdHVkeV9MaXN0JGdyb3VwZWRfY29udmVyc2lvbiksIlxuKE49Iix0YWJsZShGaW5hbF9DTFNfMjAyMV9TdHVkeV9MaXN0JGdyb3VwZWRfY29udmVyc2lvbiksIikiLHNlcD0iIikNCg0Kbl9mdW4gPC0gZnVuY3Rpb24oeCl7DQogIHJldHVybihkYXRhLmZyYW1lKHkgPSBtZWRpYW4oeCksIGxhYmVsID0gcGFzdGUwKCJuID0gIixsZW5ndGgoeCkpKSkNCn0NCg0KDQpwIDwtDQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSdTZWFyY2gnKSAlPiUgDQogIGZpbHRlcihjb252ZXJzaW9uICE9ICdEZXNrdG9wIERvd25sb2FkcycpICU+JQ0KICBmaWx0ZXIodGFjdGljIT0nQWxsJykgJT4lIA0KDQogIG11dGF0ZShjbGFzcyA9IGZjdF9yZW9yZGVyKGFzLmZhY3Rvcihncm91cGVkX2NvbnZlcnNpb24pLCBhYnNvbHV0ZV9saWZ0LCAuZnVuPSdtZWRpYW4nKSkgJT4lDQogIGdncGxvdCgpICsNCiAgICBhZXMoDQogICAgICB4ID0gcmVvcmRlcihhcy5mYWN0b3IoY2xhc3MpLGFic29sdXRlX2xpZnQpLHkgPSBhYnNvbHV0ZV9saWZ0LCBmaWxsID0gZ3JvdXBlZF9jb252ZXJzaW9uDQogICAgICAgICNhYnNvbHV0ZV9saWZ0KjEwMC9jb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXANCiAgICAgICAgKSArDQogIGdlb21fYm94cGxvdCgpICsNCnN0YXRfc3VtbWFyeShmdW4uZGF0YSA9IG5fZnVuLCBnZW9tID0gInRleHQiLCBzaXplID0gMikgKw0KICAgIHRoZW1lX2J3KCkgKw0KICAgIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbj0ibm9uZSIsDQogICAgICAgICAgYXhpcy50ZXh0Lng9ZWxlbWVudF9ibGFuaygpLA0KICAgICAgICAgICBheGlzLnRpdGxlLnkgPSBlbGVtZW50X2JsYW5rKCkNCiAgICAgICAgICApICsNCiAgICB4bGFiKCJHcm91cGVkIENvbnZlcnNpb24gVHlwZSIpKw0KICMgICAgc2NhbGVfeF9kaXNjcmV0ZShsYWJlbHM9bXlfeGxhYikgKw0KICAgIHlsYWIoIkxpZnQiKSArDQogIGNvb3JkX2ZsaXAoKQ0KDQpnZ3Bsb3RseShwLCB0b29sdGlwID0gYygidGV4dCIpKQ0KYGBgDQoNCiMjIFN1bW1hcnkgb2YgRmluZGluZ3MNCg0KMS4gVmVyeSBTaWduaWZpY2FudCBwb3NpdGl2ZSBsaWZ0IGV2ZW50IG91dGxpZXJzIGluIEZyZWUgVHJpYWwgU3VibWl0IGFuZCBOb24tTW9iaWxlIERldmljZSBDb252ZXJzaW9ucw0KDQoyLiBOZWdhdGl2ZSBsaWZ0cyBpbiBEZXNrdG9wIERvd25sb2FkcywgTm9uLU1vYmlsZSBEZXZpY2UgQ29udmVyc2lvbnMgYW5kIE1vYmlsZSBDb252ZXJzaW9ucw0KDQozLiBNdWNoIGxhcmdlciB2YXJpYXRpb24gaW4gTm9uLU1vYmlsZSBEZXZpY2UgQ29udmVyc2lvbnMgdGhhbiBNb2JpbGUgQ29udmVyc2lvbnMNCg0KNC4gRGVza3RvcCBEb3dubG9hZHMgaGFzIGxhcmdlc3QgcmFuZ2UgaW4gdmFsdWVzIHdpdGggbWluaW11bSBhdCAtMzI1MiBhbmQgbWF4aW11bSBhdCAxNzEyNjkNCg0KNS4gRGVza3RvcCBEb3dubG9hZHMgaGFkIG1heGltdW0gdmFsdWVzIG9mIGFic29sdXRlIGxpZnQgb2YgYWxsIGNvbnZlcnNpb24gZXZlbnRzLCBidXQgbG93ZXN0IHJhbmdlIG9mIGNvc3Qgc3BlbnQgd2l0aCB0aGUgZXhjZXB0aW9uIG9mIG91dGxpZXJzDQoNCjYuIE5vbi1Nb2JpbGUgRGV2aWNlIENvbnZlcnNpb25zIGNvc3QgbWVkaWFuIDMgdGltZXMgYmlnZ2VyIHRoYW4gTW9iaWxlIENvbnZlcnNpb25zIGhhZCBtdWNoIGxhcmdlciByYW5nZSBvbiBjb3N0IHNwZW50IHRoYW4gTW9iaWxlIENvbnZlcnNpb25zLCBidXQgbGlmdCB3YXMgbm90IHNpZ25pZmljYW50bHkgaGlnaGVyIGFuZCBpbmNsdWRlZCBzaWduaWZpY2FudGx5IG1vcmUgbmVnYXRpdmUgbGlmdA0KDQojI0NvdW50IG9mIG5lZ2F0aXZlIGxpZnRzIGJ5IEdyb3VwZWQgQ29udmVyc2lvbg0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAiU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICBncm91cF9ieShncm91cGVkX2NvbnZlcnNpb24pICU+JQ0KICBzdW1tYXJpc2UoTmVnYXRpdmVfQ291bnQgPSBzdW0oYWJzb2x1dGVfbGlmdCA8IDApLA0KICAgICAgICAgICAgUGVyY2VudF9OZWdhdGl2ZSA9IDEwMCAqIChzdW0oTmVnYXRpdmVfQ291bnQpL24oKSkpDQpgYGANCg0KDQoNCiMjIEJveHBsb3Qgb2YgQ29zdCBTcGVudCBieSBSZWdpb24NCmBgYHtyIGZpZy5oZWlnaHQgPSAxMCwgZmlnLndpZHRoID0gMTB9DQojbXlfeGxhYiA8LSBwYXN0ZShsZXZlbHMoRmluYWxfQ0xTXzIwMjFfU3R1ZHlfTGlzdCRncm91cGVkX2NvbnZlcnNpb24pLCJcbihOPSIsdGFibGUoRmluYWxfQ0xTXzIwMjFfU3R1ZHlfTGlzdCRncm91cGVkX2NvbnZlcnNpb24pLCIpIixzZXA9IiIpDQoNCm5fZnVuIDwtIGZ1bmN0aW9uKHgpew0KICByZXR1cm4oZGF0YS5mcmFtZSh5ID0gbWVkaWFuKHgpLCBsYWJlbCA9IHBhc3RlMCgibiA9ICIsbGVuZ3RoKHgpKSkpDQp9DQoNCg0KcCA8LQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0nU2VhcmNoJykgJT4lIA0KICBmaWx0ZXIodGFjdGljICE9ICdBbGwnKSAlPiUNCg0KICBtdXRhdGUoY2xhc3MgPSBmY3RfcmVvcmRlcihhcy5mYWN0b3IocmVnaW9uX3YyKSwgY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCAuZnVuPSdtZWRpYW4nKSkgJT4lDQogIGdncGxvdCgpICsNCiAgICBhZXMoDQogICAgICB4ID0gcmVvcmRlcihhcy5mYWN0b3IoY2xhc3MpLGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCkseSA9IGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCwgZmlsbCA9IHJlZ2lvbl92Mg0KICAgICAgICAjYWJzb2x1dGVfbGlmdCoxMDAvY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwDQogICAgICAgICkgKw0KICBnZW9tX2JveHBsb3QoKSArDQpzdGF0X3N1bW1hcnkoZnVuLmRhdGEgPSBuX2Z1biwgZ2VvbSA9ICJ0ZXh0Iiwgc2l6ZSA9IDIpICsNCiAgICB0aGVtZV9idygpICsNCiAgICB0aGVtZShsZWdlbmQucG9zaXRpb249Im5vbmUiLA0KICAgICAgICAgIGF4aXMudGV4dC54PWVsZW1lbnRfYmxhbmsoKSwNCiAgICAgICAgICAgYXhpcy50aXRsZS55ID0gZWxlbWVudF9ibGFuaygpDQogICAgICAgICAgKSArDQogICAgeGxhYigiUmVnaW9uIFR5cGUiKSsNCiAjICAgIHNjYWxlX3hfZGlzY3JldGUobGFiZWxzPW15X3hsYWIpICsNCiAgICB5bGFiKCJDb3N0IFNwZW50IG9uIEV4cG9zZWQgR3JvdXAiKSArDQogIGNvb3JkX2ZsaXAoKQ0KDQpnZ3Bsb3RseShwLCB0b29sdGlwID0gYygidGV4dCIpKQ0KYGBgDQoNCiMjIEJveHBsb3Qgb2YgQWJzb2x1dGUgTGlmdCBieSBSZWdpb24NCmBgYHtyIGZpZy5oZWlnaHQgPSAxMCwgZmlnLndpZHRoID0gMTB9DQojbXlfeGxhYiA8LSBwYXN0ZShsZXZlbHMoRmluYWxfQ0xTXzIwMjFfU3R1ZHlfTGlzdCRncm91cGVkX2NvbnZlcnNpb24pLCJcbihOPSIsdGFibGUoRmluYWxfQ0xTXzIwMjFfU3R1ZHlfTGlzdCRncm91cGVkX2NvbnZlcnNpb24pLCIpIixzZXA9IiIpDQoNCm5fZnVuIDwtIGZ1bmN0aW9uKHgpew0KICByZXR1cm4oZGF0YS5mcmFtZSh5ID0gbWVkaWFuKHgpLCBsYWJlbCA9IHBhc3RlMCgibiA9ICIsbGVuZ3RoKHgpKSkpDQp9DQoNCg0KcCA8LQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0nU2VhcmNoJykgJT4lIA0KICBmaWx0ZXIodGFjdGljICE9ICdBbGwnKSAlPiUNCg0KICBtdXRhdGUoY2xhc3MgPSBmY3RfcmVvcmRlcihhcy5mYWN0b3IocmVnaW9uX3YyKSwgYWJzb2x1dGVfbGlmdCwgLmZ1bj0nbWVkaWFuJykpICU+JQ0KICBnZ3Bsb3QoKSArDQogICAgYWVzKA0KICAgICAgeCA9IHJlb3JkZXIoYXMuZmFjdG9yKGNsYXNzKSxhYnNvbHV0ZV9saWZ0KSx5ID0gYWJzb2x1dGVfbGlmdCwgZmlsbCA9IHJlZ2lvbl92Mg0KICAgICAgICAjYWJzb2x1dGVfbGlmdCoxMDAvY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwDQogICAgICAgICkgKw0KICBnZW9tX2JveHBsb3QoKSArDQpzdGF0X3N1bW1hcnkoZnVuLmRhdGEgPSBuX2Z1biwgZ2VvbSA9ICJ0ZXh0Iiwgc2l6ZSA9IDIpICsNCiAgICB0aGVtZV9idygpICsNCiAgICB0aGVtZShsZWdlbmQucG9zaXRpb249Im5vbmUiLA0KICAgICAgICAgIGF4aXMudGV4dC54PWVsZW1lbnRfYmxhbmsoKSwNCiAgICAgICAgICAgYXhpcy50aXRsZS55ID0gZWxlbWVudF9ibGFuaygpDQogICAgICAgICAgKSArDQogICAgeGxhYigiUmVnaW9uIFR5cGUiKSsNCiAjICAgIHNjYWxlX3hfZGlzY3JldGUobGFiZWxzPW15X3hsYWIpICsNCiAgICB5bGFiKCJMaWZ0IikgKw0KICBjb29yZF9mbGlwKCkNCg0KZ2dwbG90bHkocCwgdG9vbHRpcCA9IGMoInRleHQiKSkNCmBgYA0KKioqDQojIyBCb3hwbG90IG9mIEFic29sdXRlIExpZnQgYnkgUmVnaW9uIC0gRXhjbHVkaW5nIEFPDQpgYGB7cn0NCiNteV94bGFiIDwtIHBhc3RlKGxldmVscyhGaW5hbF9DTFNfMjAyMV9TdHVkeV9MaXN0JGdyb3VwZWRfY29udmVyc2lvbiksIlxuKE49Iix0YWJsZShGaW5hbF9DTFNfMjAyMV9TdHVkeV9MaXN0JGdyb3VwZWRfY29udmVyc2lvbiksIikiLHNlcD0iIikNCg0Kbl9mdW4gPC0gZnVuY3Rpb24oeCl7DQogIHJldHVybihkYXRhLmZyYW1lKHkgPSBtZWRpYW4oeCksIGxhYmVsID0gcGFzdGUwKCJuID0gIixsZW5ndGgoeCkpKSkNCn0NCg0KDQpwIDwtDQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSdTZWFyY2gnKSAlPiUgDQogIGZpbHRlcih0YWN0aWMgIT0gJ0FsbCcpICU+JQ0KICBmaWx0ZXIocmVnaW9uX3YyICE9ICJBTyIpICU+JQ0KDQogIG11dGF0ZShjbGFzcyA9IGZjdF9yZW9yZGVyKGFzLmZhY3RvcihyZWdpb25fdjIpLCBhYnNvbHV0ZV9saWZ0LCAuZnVuPSdtZWRpYW4nKSkgJT4lDQogIGdncGxvdCgpICsNCiAgICBhZXMoDQogICAgICB4ID0gcmVvcmRlcihhcy5mYWN0b3IoY2xhc3MpLGFic29sdXRlX2xpZnQpLHkgPSBhYnNvbHV0ZV9saWZ0LCBmaWxsID0gcmVnaW9uX3YyDQogICAgICAgICNhYnNvbHV0ZV9saWZ0KjEwMC9jb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXANCiAgICAgICAgKSArDQogIGdlb21fYm94cGxvdCgpICsNCnN0YXRfc3VtbWFyeShmdW4uZGF0YSA9IG5fZnVuLCBnZW9tID0gInRleHQiLCBzaXplID0gMikgKw0KICAgIHRoZW1lX2J3KCkgKw0KICAgIHRoZW1lKGxlZ2VuZC5wb3NpdGlvbj0ibm9uZSIsDQogICAgICAgICAgYXhpcy50ZXh0Lng9ZWxlbWVudF9ibGFuaygpLA0KICAgICAgICAgICBheGlzLnRpdGxlLnkgPSBlbGVtZW50X2JsYW5rKCkNCiAgICAgICAgICApICsNCiAgICB4bGFiKCJSZWdpb24gVHlwZSIpKw0KICMgICAgc2NhbGVfeF9kaXNjcmV0ZShsYWJlbHM9bXlfeGxhYikgKw0KICAgIHlsYWIoIkxpZnQiKSArDQogIGNvb3JkX2ZsaXAoKQ0KDQpnZ3Bsb3RseShwLCB0b29sdGlwID0gYygidGV4dCIpKQ0KYGBgDQoNCiMjIFN1bW1hcnkgb2YgRmluZGluZ3MNCg0KMS4gVVMgKyBDQSBzcGVudCBvbiBhIG11Y2ggbGFyZ2VyIHNjYWxlIHRoYW4gdGhlIHJlc3Qgb2YgY291bnRyaWVzIGluIEFNRVIsIHlldCBvYnRhaW5lZCBuZWdhdGl2ZSBsaWZ0IHZhbHVlcyBhbmQgYW4gb3ZlcmFsbCBsb3dlciByYW5nZSB0aGFuIEFNRVINCg0KMi5VUyArIENBIGNvc3QgbWVkaWFuIG92ZXIgdGVuIHRpbWVzIG1lZGlhbiBvZiB0aGUgcmVzdCBvZiBBTUVSIGNvdW50cmllcw0KDQozLiBVUyArIENBIGFuZCBFTUVBIHJlZ2lvbnMgaGFkIG5lZ2F0aXZlIGxpZnRzIA0KDQojI0NvdW50IG9mIG5lZ2F0aXZlIGxpZnRzIGJ5IFJlZ2lvbg0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAiU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICBncm91cF9ieShyZWdpb25fdjIpICU+JQ0KICBzdW1tYXJpc2UoTmVnYXRpdmVfQ291bnQgPSBzdW0oYWJzb2x1dGVfbGlmdCA8IDApLA0KICAgICAgICAgICAgUGVyY2VudF9OZWdhdGl2ZSA9IDEwMCAqIChzdW0oTmVnYXRpdmVfQ291bnQpL24oKSkpDQpgYGANCg0KIyMgU3VtbWFyaXplIFNwZW5kDQoNCiMjIyBPdmVyYWxsIFNwZW5kIC0gYm90aCBTZWFyY2ggYW5kIE5vbi1TZWFyY2gNCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUgDQogIHN1bW1hcmlzZShPdmVyYWxsX3NwZW5kID0gc3VtKGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCksDQogICAgICAgICAgICBPdmVyYWxsX1N0dWRpZXMgPSBuX2Rpc3RpbmN0KHN0dWR5X2lkLHF1YXJ0ZXIpDQogICAgICAgICAgICApIA0KYGBgDQoqKioNCiMjIyBTZWFyY2ggRGF0YQ0KDQojIyMjIE92ZXJhbGwgU3BlbmQgDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lIA0KICBmaWx0ZXIoY2hhbm5lbCA9PSAnU2VhcmNoJykgJT4lIA0KICBmaWx0ZXIodGFjdGljICE9ICdBbGwnKSAlPiUgI2ZpbHRlcmluZyB3aGVyZSB0YWN0aWMgIT0gQWxsIHRvIGV4Y2x1ZGUgb3ZlcmFsbCBzcGVuZCBmb3IgZWFjaCBzdHVkeS4gY2FuIGFsc28gdXNlIHRhY3RpYyA9PSAnQWxsJyB0byBjb25maXJtIHRvdGFscyBhZGQgdXAgDQogIHN1bW1hcmlzZShPdmVyYWxsX3NwZW5kID0gc3VtKGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCksDQogICAgICAgICAgICBPdmVyYWxsX1N0dWRpZXMgPSBuX2Rpc3RpbmN0KHN0dWR5X2lkLHF1YXJ0ZXIpDQogICAgICAgICAgICApIA0KYGBgDQoqKioNCiMjIyMgU3BlbmQgcGVyIFN0dWR5IA0KU3R1ZHkgMjIgaGFkIG1heGltdW0gc3BlbmQgYXQgb3ZlciA1IG1pbGxpb24NCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0gIlNlYXJjaCIpICU+JQ0KICBmaWx0ZXIodGFjdGljID09ICJBbGwiKSAlPiUNCiAgc2VsZWN0KHN0dWR5X2lkLCBjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXApDQpgYGANCg0KIyMjIyBPdmVyYWxsIFNwZW5kIGJ5IFBBIA0KDQojIyMjU2lnbmlmaWNhbnRseSBtb3JlIHdhcyBzcGVudCBvbiBHb29nbGUgQ2xvdWQNCk1heGltdW0gU3BlbmQgb24gR29vZ2xlIENsb3VkIGFuZCBtaW5pbXVtIG9uIENocm9tZWJvb2sNCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUgDQogIGZpbHRlcihjaGFubmVsID09ICdTZWFyY2gnKSAlPiUgDQogIGZpbHRlcih0YWN0aWMgIT0gJ0FsbCcpICU+JQ0KICBncm91cF9ieShwYSkgJT4lIA0KICBzdW1tYXJpc2UoT3ZlcmFsbF9zcGVuZCA9IHN1bShjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXApLA0KICAgICAgICAgICAgT3ZlcmFsbF9TdHVkaWVzID0gbl9kaXN0aW5jdChzdHVkeV9pZCxxdWFydGVyKSwNCiAgICAgICAgICAgIEF2ZXJhZ2VfU3BlbmQgPSBtZWFuKGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCksDQogICAgICAgICAgICBPdmVyYWxsX2luc3RhbmNlcyA9IG4oKQ0KICAgICAgICAgICAgKSANCmBgYA0KDQoNCg0KKioqDQojIyMgU2lnbmlmaWNhbnQgU3R1ZHkgJSAtIFNlYXJjaCBEYXRhDQoNCiMjIyMgQWJvdXQgOTIlIG9mIFNlYXJjaCBzcGVuZCBpcyBTaWduaWZpY2FudCANCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUgDQogIGZpbHRlcihjaGFubmVsID09ICdTZWFyY2gnKSAlPiUgDQogIGZpbHRlcih0YWN0aWMgIT0gJ0FsbCcpICU+JSANCiAgZ3JvdXBfYnkoU2lnbmlmaWNhbnRfU3BlbmQpICU+JQ0KICBzdW1tYXJpc2UoT3ZlcmFsbF9zcGVuZCA9IHN1bShjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXApKSU+JSANCiAgbXV0YXRlKFBlcmNlbnRfc3BlbmQgPSAxMDAgKiBPdmVyYWxsX3NwZW5kLyhzdW0oT3ZlcmFsbF9zcGVuZCkpKQ0KYGBgDQoqKioNCg0KDQoNCiMjIyMgU2lnbmlmaWNhbnQgU3BlbmQgYnkgUEEgDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09ICdTZWFyY2gnKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAnQWxsJykgJT4lDQogIGdyb3VwX2J5KHBhLCBTaWduaWZpY2FudF9TcGVuZCkgJT4lDQogIHN1bW1hcmlzZShPdmVyYWxsX3NwZW5kID0gc3VtKGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCkpJT4lIA0KICBtdXRhdGUoUGVyY2VudF9zcGVuZCA9IDEwMCAqIE92ZXJhbGxfc3BlbmQvKHN1bShPdmVyYWxsX3NwZW5kKSkpDQpgYGANCioqKg0KIyMjIyBHcm91cGVkIEJhciBDaGFydCBmb3IgU2lnbmlmaWNhbnQgU3BlbmQgYnkgUEENCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0gJ1NlYXJjaCcpICU+JQ0KICBmaWx0ZXIodGFjdGljICE9ICdBbGwnKSAlPiUNCiAgZ3JvdXBfYnkocGEsIFNpZ25pZmljYW50X1NwZW5kKSAlPiUNCiAgc3VtbWFyaXNlKE92ZXJhbGxfc3BlbmQgPSBzdW0oY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwKSklPiUgDQogIGdncGxvdChhZXMoeD1wYSwgeT1PdmVyYWxsX3NwZW5kLCBmaWxsPWFzLmZhY3RvcihTaWduaWZpY2FudF9TcGVuZCkpKSArIA0KICBnZW9tX2Jhcihwb3NpdGlvbj0iZG9kZ2UiLCBzdGF0PSJpZGVudGl0eSIpICsNCiAgeGxhYigiUEEiKSArDQogIHlsYWIoIlRvdGFsIFNwZW5kIikgKw0KICBnZ3RpdGxlKCJTaWduaWZpY2FudCBTcGVuZCBieSBQQSIpICsNCiAgY29vcmRfZmxpcCgpICsNCiAgdGhlbWVfbWluaW1hbCgpDQogIA0KYGBgDQoNCioqKg0KDQojIyMjIEdyb3VwZWQgQmFyIENoYXJ0IFNpZ25pZmljYW50IFN0dWRpZXMgYnkgUEEgLSBleGNsdWRpbmcgUEEncyB0aGF0IGFyZSAxMDAlIFNpZ25pZmljYW50IC0gQ2hyb21lYm9vayAmIEdvb2dsZSBDbG91ZA0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KIGZpbHRlcihjaGFubmVsID09ICdTZWFyY2gnKSAlPiUNCiBmaWx0ZXIodGFjdGljICE9ICdBbGwnKSAlPiUNCiBmaWx0ZXIoIXBhICVpbiUgYygiQ2hyb21lYm9vayIsICJHb29nbGUgQ2xvdWQiKSkgJT4lDQogZ3JvdXBfYnkocGEsIFNpZ25pZmljYW50X1NwZW5kKSAlPiUNCiAgc3VtbWFyaXNlKE92ZXJhbGxfc3BlbmQgPSBzdW0oY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwKSklPiUgDQogIGdncGxvdChhZXMoeD1wYSwgeT1PdmVyYWxsX3NwZW5kLCBmaWxsPWFzLmZhY3RvcihTaWduaWZpY2FudF9TcGVuZCkpKSArIA0KICBnZW9tX2Jhcihwb3NpdGlvbj0iZG9kZ2UiLCBzdGF0PSJpZGVudGl0eSIpICsNCiAgeGxhYigiUEEiKSArDQogIHlsYWIoIlRvdGFsIFNwZW5kIikgKw0KICBnZ3RpdGxlKCJTcGVuZCBieSBQQSBhbmQgU2lnbmlmaWNhbnQgU3BlbmQgRXhjbHVkaW5nIENocm9tZWJvb2sgJiBHb29nbGUgQ2xvdWQiKSArDQogIGNvb3JkX2ZsaXAoKSArDQogIHRoZW1lX21pbmltYWwoKQ0KYGBgDQoqKioNCiMjIyBTdW1tYXJpemUgRmluZGluZ3MNCjEuIEFsbCBvZiBzcGVuZCBvbiBHb29nbGUgQ2xvdWQgYW5kIENocm9tZWJvb2sgd2FzIHNpZ25pZmljYW50DQoNCjIuIFBpeGVsIHdhcyB0aGUgb25seSBQQSB3aXRoIG1ham9yaXR5IG9mIHNwZW5kIGFzIG5vbi1zaWduaWZpY2FudCwga2VlcCBpbiBtaW5kIGl0IG9ubHkgYXBwZWFyZWQgaW4gNyBpbnN0YW5jZXMgb2YgdGhlIGRhdGEgDQoNCjMuIEdvb2dsZSBDbG91ZCBhdmVyYWdlZCBoaWdoZXN0IHNwZW5kIGFuZCBoYWQgbGFyZ2VzdCBvdmVyYWxsIHNwZW5kIA0KDQojIyMjIFNpZ25pZmljYW50IFNwZW5kIGJ5IFJlZ2lvbg0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAnU2VhcmNoJykgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gJ0FsbCcpICU+JQ0KICBncm91cF9ieShyZWdpb25fdjIsIFNpZ25pZmljYW50X1NwZW5kKSAlPiUNCiAgc3VtbWFyaXNlKE92ZXJhbGxfc3BlbmQgPSBzdW0oY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwKSklPiUgDQogIG11dGF0ZShQZXJjZW50X3NwZW5kID0gMTAwICogT3ZlcmFsbF9zcGVuZC8oc3VtKE92ZXJhbGxfc3BlbmQpKSkNCmBgYA0KIyMjIyBCYXIgQ2hhcnQgb2YgU2lnbmlmaWNhbnQgU3BlbmQgYnkgUmVnaW9uDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09ICdTZWFyY2gnKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAnQWxsJykgJT4lDQogIGdyb3VwX2J5KHJlZ2lvbl92MiwgU2lnbmlmaWNhbnRfU3BlbmQpICU+JQ0KICBzdW1tYXJpc2UoT3ZlcmFsbF9zcGVuZCA9IHN1bShjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXApKSAlPiUgDQogIGdncGxvdChhZXMoeD1yZWdpb25fdjIsIHk9T3ZlcmFsbF9zcGVuZCwgZmlsbD1hcy5mYWN0b3IoU2lnbmlmaWNhbnRfU3BlbmQpKSkgKyANCiAgZ2VvbV9iYXIocG9zaXRpb249ImRvZGdlIiwgc3RhdD0iaWRlbnRpdHkiKSArDQogIHhsYWIoIlJlZ2lvbiIpICsNCiAgeWxhYigiVG90YWwgU3BlbmQiKSArDQogIGdndGl0bGUoIlNwZW5kIGJ5IFJlZ2lvbiBhbmQgU2lnbmlmaWNhbnQgU3BlbmQiKSArDQogIGNvb3JkX2ZsaXAoKSArDQogIHRoZW1lX21pbmltYWwoKQ0KYGBgDQoqKioNCg0KDQpOb24tU2lnbmlmaWNhbnQgU3R1ZGllcyB0b28gbG93IHRvIGJlIHNlZW4gb24gYmFycGxvdA0KIyMjIyBCYXIgQ2hhcnQgb2YgU2lnbmlmaWNhbnQgU3BlbmQgYnkgUmVnaW9uIC0gRXhjbHVkaW5nIFVTICsgQ0ENCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0gJ1NlYXJjaCcpICU+JQ0KICBmaWx0ZXIodGFjdGljICE9ICdBbGwnKSAlPiUNCiAgZmlsdGVyKCFyZWdpb25fdjIgJWluJSBjKCdBTUVSX1VTQ0EnKSkgJT4lDQogIGdyb3VwX2J5KHJlZ2lvbl92MiwgU2lnbmlmaWNhbnRfU3BlbmQpICU+JQ0KICBzdW1tYXJpc2UoT3ZlcmFsbF9zcGVuZCA9IHN1bShjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXApKSU+JSANCiAgZ2dwbG90KGFlcyh4PXJlZ2lvbl92MiwgeT1PdmVyYWxsX3NwZW5kLCBmaWxsPWFzLmZhY3RvcihTaWduaWZpY2FudF9TcGVuZCkpKSArIA0KICBnZW9tX2Jhcihwb3NpdGlvbj0iZG9kZ2UiLCBzdGF0PSJpZGVudGl0eSIpICsNCiAgeGxhYigiUmVnaW9uIikgKw0KICB5bGFiKCJUb3RhbCBTcGVuZCIpICsNCiAgZ2d0aXRsZSgiU3BlbmQgYnkgUmVnaW9uIGFuZCBTaWduaWZpY2FudCBTcGVuZCBFeGNsdWRpbmcgQ0EgJiBVUyIpICsNCiAgY29vcmRfZmxpcCgpICsNCiAgdGhlbWVfbWluaW1hbCgpDQpgYGANCg0KIyMjIyBTaWduaWZpY2FudCBTcGVuZCBieSBUYWN0aWMgDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09ICdTZWFyY2gnKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAnQWxsJykgJT4lDQogIGdyb3VwX2J5KHRhY3RpYywgU2lnbmlmaWNhbnRfU3BlbmQpICU+JQ0KICBzdW1tYXJpc2UoT3ZlcmFsbF9zcGVuZCA9IHN1bShjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXApKSU+JSANCiAgbXV0YXRlKFBlcmNlbnRfc3BlbmQgPSAxMDAgKiBPdmVyYWxsX3NwZW5kLyhzdW0oT3ZlcmFsbF9zcGVuZCkpKQ0KYGBgDQojIyMjIEJhciBjaGFydCBvZiBTaWduaWZpY2FudCBTcGVuZCBieSBUYWN0aWMgDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09ICdTZWFyY2gnKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAnQWxsJykgJT4lDQogIGdyb3VwX2J5KHRhY3RpYywgU2lnbmlmaWNhbnRfU3BlbmQpICU+JQ0KICBzdW1tYXJpc2UoT3ZlcmFsbF9zcGVuZCA9IHN1bShjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXApKSU+JSANCiAgZ2dwbG90KGFlcyh4PXRhY3RpYywgeT1PdmVyYWxsX3NwZW5kLCBmaWxsPWFzLmZhY3RvcihTaWduaWZpY2FudF9TcGVuZCkpKSArIA0KICBnZW9tX2Jhcihwb3NpdGlvbj0iZG9kZ2UiLCBzdGF0PSJpZGVudGl0eSIpICsNCiAgeGxhYigiVGFjdGljIikgKw0KICB5bGFiKCJUb3RhbCBTcGVuZCIpICsNCiAgZ2d0aXRsZSgiU2lnbmlmaWNhbnQgU3BlbmQgYnkgVGFjdGljIikgKw0KICBjb29yZF9mbGlwKCkgKw0KICB0aGVtZV9taW5pbWFsKCkNCiAgDQpgYGANCiMjIyBTdW1tYXJ5IG9mIEZpbmRpbmdzDQoxLiBBbGwgcmVnaW9ucyB3aXRoIHRoZSBleGNlcHRpb24gb2YgQ0EgKyBVUyB3ZXJlIDk5JSsgU2lnbmlmaWNhbnQsIENBICsgVVMgd2VyZSA4NCUgc2lnbmlmaWNhbnQNCg0KMi4gU2lnbmlmaWNhbnQgU3BlbmQgJSB3aXRoaW4gU0tXUyB3YXMgaGlnaGVzdCB3aGVuIGNvbXBhcmVkIHRvIG90aGVyIHRhY3RpY3MgDQoNCiMjIzNEIFNjYXR0ZXJwbG90cyB0byBzZWUgcmVsYXRpb25zaGlwcyBiZXR3ZWVuIHNwZW5kLCBkdXJhdGlvbiBhbmQgbGlmdA0KYGBge3J9DQojaW5zdGFsbC5wYWNrYWdlcygic2NhdHRlcnBsb3QzZCIpDQojaW5zdGFsbC5wYWNrYWdlcygicmdsIikNCiNpbnN0YWxsLnBhY2thZ2VzKCJyZGx3aWRnZXQiKQ0KI2xpYnJhcnkoc2NhdHRlcnBsb3QzZCkNCiNsaWJyYXJ5KCJyZ2wiKQ0KDQoNCiNTY2F0dGVyM0RfR3JhcGggPC0gRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCU+JQ0KICMgZmlsdGVyKGNoYW5uZWwgPT0gJ1NlYXJjaCcpICU+JQ0KICAjZmlsdGVyKHRhY3RpYyAhPSdBbGwnKQ0KICANCnNjYXR0ZXJwbG90M2QoU2NhdHRlcjNEX0dyYXBoJGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCwNCiAgICAgICAgICAgICAgU2NhdHRlcjNEX0dyYXBoJGR1cmF0aW9uLA0KICAgICAgICAgICAgICBTY2F0dGVyM0RfR3JhcGgkYWJzb2x1dGVfbGlmdCwNCiAgICAgICAgICAgICAgbWFpbj0iM0QgU2NhdHRlcnBsb3QiLA0KICAgICAgICAgICAgICBhbmdsZT01NSwNCiAgICAgICAgICAgICAgeGxhYiA9ICJDb3N0IFNwZW50IiwNCiAgICAgICAgICAgICAgeWxhYiA9ICJEdXJhdGlvbiIsDQogICAgICAgICAgICAgIHpsYWIgPSAiTGlmdCIpDQoNCnBsb3QzZChTY2F0dGVyM0RfR3JhcGgkY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLA0KICAgICAgICAgICAgICBTY2F0dGVyM0RfR3JhcGgkZHVyYXRpb24sDQogICAgICAgICAgICAgIFNjYXR0ZXIzRF9HcmFwaCRhYnNvbHV0ZV9saWZ0LA0KICAgICAgICAgICAgICBtYWluPSIzRCBTY2F0dGVycGxvdCIsDQogICAgICAgICAgICAgIGFuZ2xlPTU1LA0KICAgICAgICAgICAgICB4bGFiID0gIkNvc3QgU3BlbnQiLA0KICAgICAgICAgICAgICB5bGFiID0gIkR1cmF0aW9uIiwNCiAgICAgICAgICAgICAgemxhYiA9ICJMaWZ0IikNCmBgYA0KDQoNCiMjU2NhdHRlcnBsb3Qgb2YgVG90YWwgQ29zdCBTcGVudCB2cy4gQWJzb2x1dGUgTGlmdCBmb3IgYWxsIFNlYXJjaA0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSdTZWFyY2gnKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAnQWxsJyklPiUNCiAgZ2dwbG90KGFlcyh4ID0gY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCB5ID0gYWJzb2x1dGVfbGlmdCkpICsNCiAgZ2VvbV9wb2ludCgpICsNCiAgICAgICB4bGFiKCJDb3N0IFNwZW50IikgKw0KICAgICAgIHlsYWIoIkxpZnQiKSArDQogICAgICAgZ2d0aXRsZSgiQ29zdCB2cy4gTGlmdCIpDQpgYGANCiNTY2F0dGVycGxvdCBvZiBUb3RhbCBDb3N0IHZzLiBBYnNvbHV0ZSBMaWZ0LCBmb3IgZWFjaCBpbmRpdmlkdWFsIHN0dWR5IA0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSdTZWFyY2gnKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyA9PSAnQWxsJyklPiUNCiAgZ2dwbG90KGFlcyh4ID0gY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCB5ID0gYWJzb2x1dGVfbGlmdCkpICsNCiAgZ2VvbV9wb2ludCgpICsNCiAgICAgICB4bGFiKCJDb3N0IFNwZW50IikgKw0KICAgICAgIHlsYWIoIkxpZnQiKSArDQogICAgICAgZ2d0aXRsZSgiQ29zdCB2cy4gTGlmdCIpDQpgYGANCg0KDQoNCiMjU2NhdHRlcnBsb3Qgb2YgQ29zdCBTcGVudCB2cy4gQWJzb2x1dGUgTGlmdCwgYnkgUmVnaW9uDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09J1NlYXJjaCcpICU+JQ0KICBmaWx0ZXIodGFjdGljICE9ICdBbGwnKSU+JQ0KICBnZ3Bsb3QoYWVzKHggPSBjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXAsIHkgPSBhYnNvbHV0ZV9saWZ0LCBjb2xvciA9IHJlZ2lvbl92MikpICsNCiAgZ2VvbV9wb2ludCgpICsNCiAgICAgICB4bGFiKCJDb3N0IFNwZW50IikgKw0KICAgICAgIHlsYWIoIkxpZnQiKSArDQogICAgICAgZ2d0aXRsZSgiQ29zdCB2cy4gTGlmdCIpDQpgYGANCg0KIyNTY2F0dGVycGxvdCBvZiBDb3N0IFNwZW50IHZzLiBBYnNvbHV0ZSBMaWZ0LCBieSBDb252ZXJzaW9uIEdyb3VwDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09J1NlYXJjaCcpICU+JQ0KICBmaWx0ZXIodGFjdGljICE9ICdBbGwnKSU+JQ0KICBnZ3Bsb3QoYWVzKHggPSBjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXAsIHkgPSBhYnNvbHV0ZV9saWZ0LCBjb2xvciA9IGdyb3VwZWRfY29udmVyc2lvbikpICsNCiAgZ2VvbV9wb2ludCgpICsNCiAgICAgICB4bGFiKCJDb3N0IFNwZW50IikgKw0KICAgICAgIHlsYWIoIkxpZnQiKSArDQogICAgICAgZ2d0aXRsZSgiQ29zdCB2cy4gTGlmdCIpDQpgYGANCiMjIFN1bW1hcnkgb2YgRmluZGluZ3MNCjEuIE1ham9yaXR5IG9mIHN0dWRpZXMgd2l0aCBsb3cgY29zdCBhbmQgaGlnaCBsaWZ0IGFyZSBEZXNrdG9wIERvd25sb2FkcyANCg0KMi4gVVMgKyBDQSB0ZW5kcyB0byBiZSBoaWdoIGNvc3QsIGxvdyBsaWZ0IA0KDQozLiBEZXNrdG9wIERvd25sb2FkcyBjb252ZXJzaW9uIGhhcyBsb3dlc3Qgc3BlbmQgd2l0aCBoaWdoZXN0IGxpZnQgDQoNCiMjU2NhdHRlcnBsb3Qgb2YgQ29zdCBTcGVudCB2cy4gQWJzb2x1dGUgTGlmdCwgYnkgWWVhcg0KDQojIyMgR2VuZXJhbGx5LCAyMDIyIGhhcyBoaWdoZXIgcmFuZ2UgaW4gc3BlbmQgd2l0aCBhIGxhcmdlIHJhbmdlIGluIGxpZnQgdGhhbiAyMDIxIA0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSdTZWFyY2gnKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAnQWxsJyklPiUNCiAgZ2dwbG90KGFlcyh4ID0gY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCB5ID0gYWJzb2x1dGVfbGlmdCwgY29sb3IgPSBhcy5mYWN0b3IoeWVhcikpKSArDQogIGdlb21fcG9pbnQoKSArDQogICAgICAgeGxhYigiQ29zdCBTcGVudCIpICsNCiAgICAgICB5bGFiKCJMaWZ0IikgKw0KICAgICAgIGdndGl0bGUoIkNvc3QgdnMuIExpZnQiKQ0KYGBgDQoNCiMjU2NhdHRlcnBsb3Qgb2YgQ29zdCBTcGVudCB2cyBBYnNvbHV0ZSBMaWZ0LCBieSBTaWduaWZpY2FudCBTcGVuZCANCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0nU2VhcmNoJykgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gJ0FsbCcpJT4lDQogIGdncGxvdChhZXMoeCA9IGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCwgeSA9IGFic29sdXRlX2xpZnQsIGNvbG9yID0gYXMuZmFjdG9yKFNpZ25pZmljYW50X1NwZW5kKSkpICsNCiAgZ2VvbV9wb2ludCgpICsNCiAgICAgICB4bGFiKCJDb3N0IFNwZW50IikgKw0KICAgICAgIHlsYWIoIkxpZnQiKSArDQogICAgICAgZ2d0aXRsZSgiQ29zdCB2cy4gTGlmdCBTZWFyY2giKQ0KYGBgDQoNCiMjU2NhdHRlcnBsb3Qgb2YgRHVyYXRpb24gdnMuIENvc3QgU3BlbnQgZm9yIGFsbCBTZWFyY2gNCiMjIyBOb3QgYXMgbGluZWFyIGFzIGV4cGVjdGVkIA0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSdTZWFyY2gnKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyA9PSAnQWxsJyklPiUNCiAgZ2dwbG90KGFlcyh4ID0gZHVyYXRpb24sIHkgPSBjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXApKSArDQogIGdlb21fcG9pbnQoKSArDQogICAgICAgeGxhYigiRHVyYXRpb24iKSArDQogICAgICAgeWxhYigiQ29zdCIpICsNCiAgICAgICBnZ3RpdGxlKCJEdXJhdGlvbiB2cy4gQ29zdCIpDQoNCiNHZXR0aW5nIHN0dWR5IHdpdGggbWF4IGNvc3QgDQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAiU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgPT0gIkFsbCIpICU+JQ0KICBmaWx0ZXIoY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwID09IG1heChjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXApKSAlPiUNCiAgc2VsZWN0KHN0dWR5X2lkKQ0KDQojR2V0dGluZyBzdHVkeSB3aXRoIG1heCBkdXJhdGlvbg0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0gIlNlYXJjaCIpICU+JQ0KICBmaWx0ZXIodGFjdGljID09ICJBbGwiKSAlPiUNCiAgZmlsdGVyKGR1cmF0aW9uID09IG1heChkdXJhdGlvbikpICU+JQ0KICBzZWxlY3Qoc3R1ZHlfaWQpDQoNCmBgYA0KDQojI1NjYXR0ZXJwbG90IG9mIER1cmF0aW9uIHZzLiBMaWZ0LCBieSBSZWdpb24NCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0nU2VhcmNoJykgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gJ0FsbCcpJT4lDQogIGdncGxvdChhZXMoeCA9IGR1cmF0aW9uLCB5ID0gYWJzb2x1dGVfbGlmdCwgY29sb3IgPSByZWdpb25fdjIpKSArDQogIGdlb21fcG9pbnQoKSArDQogICAgICAgeGxhYigiRHVyYXRpb24iKSArDQogICAgICAgeWxhYigiTGlmdCIpICsNCiAgICAgICBnZ3RpdGxlKCJEdXJhdGlvbiB2cy4gTGlmdCIpDQpgYGANCg0KIyNTY2F0dGVycGxvdCBvZiBEdXJhdGlvbiB2cy4gTGlmdCwgYnkgUEENCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0nU2VhcmNoJykgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gJ0FsbCcpJT4lDQogIGdncGxvdChhZXMoeCA9IGR1cmF0aW9uLCB5ID0gYWJzb2x1dGVfbGlmdCwgY29sb3IgPSBwYSkpICsNCiAgZ2VvbV9wb2ludCgpICsNCiAgICAgICB4bGFiKCJEdXJhdGlvbiIpICsNCiAgICAgICB5bGFiKCJMaWZ0IikgKw0KICAgICAgIGdndGl0bGUoIkR1cmF0aW9uIHZzLiBMaWZ0IikgDQpgYGANCiMjIFN1bW1hcnkgb2YgRmluZGluZ3MNCg0KMS5TdHVkeSAyMiBoYXMgbWF4IGNvc3Qgc3BlbnQsIHlldCBvbiB0aGUgbG93ZXIgcmFuZ2Ugb2YgZHVyYXRpb24NCg0KMi5TdHVkeSAxIGhhcyBtYXggZHVyYXRpb24gYW5kIHZlcnkgbG93IGNvc3QgDQoNCiMjIFNjYXR0ZXJwbG90cyBvbiBJbmRpdmlkdWFsIFBBJ3MNCiMjIyBTY2F0dGVycGxvdHMgb2YgQ29zdCBTcGVudCB2cy4gTGlmdCBmb3IgZWFjaCBQQQ0KYGBge3J9DQpwYV9wbG90IDwtIGZ1bmN0aW9uKHBhMikgew0KICBwbG90X3NjYXR0ZXIgPC0gRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgICBmaWx0ZXIoY2hhbm5lbCA9PSAnU2VhcmNoJykgJT4lDQogICAgZmlsdGVyKHRhY3RpYyAhPSAnQWxsJykgJT4lDQogICAgZmlsdGVyKHBhID09IHBhMikgJT4lDQogICAgZ2dwbG90KGFlcyh4ID0gY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCB5ID0gYWJzb2x1dGVfbGlmdCwgY29sb3IgPXJlZ2lvbl92MikpICsNCiAgICBnZW9tX3BvaW50KCkgKw0KICAgICAgIHhsYWIoIkNvc3QiKSArDQogICAgICAgeWxhYigiTGlmdCIpICsNCiAgICAgICBnZ3RpdGxlKHBhc3RlKHBhMiwgIiBDb3N0IHZzLiBMaWZ0IikpDQpwcmludChwbG90X3NjYXR0ZXIpDQp9DQoNCnBhX2NvbHVtbnMgPC0gYygiQ2hyb21lIiwgIkNocm9tZWJvb2siLCAiRFNNIiwgIkdvb2dsZSBDbG91ZCIsICJQaXhlbCIpDQoNCmZvciAoaSBpbiBwYV9jb2x1bW5zKSB7DQogIHBhX3Bsb3QoaSkNCn0gDQoNCg0KYGBgDQojIyBTdW1tYXJ5IG9mIEZpbmRpbmdzDQoxLiBNb3N0IGxpbmVhciB0cmVuZHMgc2VlbSB0byBiZSBDaHJvbWVib29rLCBEU00gYW5kIEdvb2dsZSBDbG91ZA0KDQoyLiBEU00gb25seSBjb250YWlucyBBTUVSIHJlZ2lvbnMNCg0KMy4gQ2hyb21lIGNvc3RzIHJlbWFpbiBsb3cgd2l0aCBoaWdoIGxpZnQsIHBlcmZvcm1lZCB3b3JzdCBpbiBVUyArIENBIHJlZ2lvbiANCg0KNC4gRm9yIENocm9tZSwgQ2hyb21lYm9vayBhbmQgR29vZ2xlIENsb3VkLCBpdCBnZW5lcmFsbHkgcmVxdWlyZXMgaGlnaGVyIGNvc3QgZm9yIGxpZnQgaW4gVVMgKyBDQSByZWdpb25zDQoNCiMjIyBTY2F0dGVycGxvdHMgb2YgQ29zdCBTcGVudCB2cy4gTGlmdCBmb3IgZWFjaCBSZWdpb24NCmBgYHtyfQ0KcmVnaW9uX3Bsb3QgPC0gZnVuY3Rpb24ocmVnaW9uMikgew0KICBwbG90X3NjYXR0ZXIgPC0gRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgICBmaWx0ZXIoY2hhbm5lbCA9PSAnU2VhcmNoJykgJT4lDQogICAgZmlsdGVyKHRhY3RpYyAhPSAnQWxsJykgJT4lDQogICAgZmlsdGVyKHJlZ2lvbl92MiA9PSByZWdpb24yKSAlPiUNCiAgICBnZ3Bsb3QoYWVzKHggPSBjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXAsIHkgPSBhYnNvbHV0ZV9saWZ0LCBjb2xvciA9IHBhKSkgKw0KICAgIGdlb21fcG9pbnQoKSArDQogICAgICAgeGxhYigiQ29zdCIpICsNCiAgICAgICB5bGFiKCJMaWZ0IikgKw0KICAgICAgIGdndGl0bGUocGFzdGUocmVnaW9uMiwgIiBDb3N0IHZzLiBMaWZ0IikpDQpwcmludChwbG90X3NjYXR0ZXIpDQp9DQoNCg0KcmVnaW9uX2NvbHMgPC0gYygiQU1FUiIsIkFNRVJfVVNDQSIsICJBTyIsICJBUEFDIiwgIkVNRUEiKQ0KZm9yIChpIGluIHJlZ2lvbl9jb2xzKSB7DQogIHJlZ2lvbl9wbG90KGkpDQp9DQpgYGANCioqKg0KIyMgU3VtbWFyeSBvZiBGaW5kaW5ncw0KMS4gRU1FQSByZWdpb24gaGFzIGhpZ2hlc3QgbGlmdCB3aXRoIGxvdyBjb3N0cw0KDQoyLiBJbiBhbGwgcmVnaW9ucyBDaHJvbWUgc2VlbWVkIHRvIGJlIGhpZ2hlc3QgcGVyZm9ybWluZywgd2l0aCBsb3cgY29zdHMgcmVzdWx0aW5nIGluIGhpZ2ggbGlmdCANCg0KMy4gVVMgKyBDQSBjb3N0IHZzLiBsaWZ0IHJlbGF0aW9uc2hpcCBtdWNoIG1vcmUgZGlzcGVyc2VkIHRoYW4gcmVzdCBvZiBBTUVSIHJlZ2lvbg0KDQo0LiBHb29nbGUgQ2xvdWQgc2VlbXMgdG8gcmVxdWlyZSBzaWduaWZpY2FudCBjb3N0IGZvciBsaWZ0IGFjcm9zcyBhbGwgcmVnaW9ucyB3aGVuIGNvbXBhcmVkIHRvIGFsbCBwYSdzIA0KDQojIyBDaHJvbWUgZGF0YQ0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSdTZWFyY2gnKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAnQWxsJyklPiUNCiAgZmlsdGVyKHBhID09ICdDaHJvbWUnKSAlPiUNCiAgc3VtbWFyeSgpDQpgYGANCiAqKioNCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0nU2VhcmNoJykgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gJ0FsbCcpJT4lDQogIGZpbHRlcihwYSA9PSAnQ2hyb21lJykgJT4lIA0KICBnZ3Bsb3QoYWVzKHggPSBkdXJhdGlvbiwgeSA9IGFic29sdXRlX2xpZnQpKSArDQogIGdlb21fcG9pbnQoKSArDQogICAgICAgeGxhYigiRHVyYXRpb24iKSArDQogICAgICAgeWxhYigiTGlmdCIpICsNCiAgICAgICBnZ3RpdGxlKCJEdXJhdGlvbiB2cy4gTGlmdCIpDQpgYGANCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0nU2VhcmNoJykgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gJ0FsbCcpJT4lDQogIGZpbHRlcihwYSA9PSAnQ2hyb21lJykgJT4lIA0KICBnZ3Bsb3QoYWVzKHggPSBkdXJhdGlvbiwgeSA9IGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCkpICsNCiAgZ2VvbV9wb2ludCgpICsNCiAgICAgICB4bGFiKCJEdXJhdGlvbiIpICsNCiAgICAgICB5bGFiKCJDb3N0IikgKw0KICAgICAgIGdndGl0bGUoIkR1cmF0aW9uIHZzLiBDb3N0IikNCmBgYA0KIyNTdW1tYXJ5IG9mIEZpbmRpbmdzDQoxLiBEdXJhdGlvbiBzZWVtcyB0byBiZSBlaXRoZXIgc2hvcnQgb3IgbG9uZywgYW5kIGRvZXNudCBzZWVtIHRvIGhhdmUgYW55IGFwcGFyZW50IGVmZmVjdCBvbiBjb3N0IG9yIGxpZnQsIHdpdGggb25seSBhIHNsaWdodCBpbmNyZWFzZSBpbiBsaWZ0IGZvciBsb25nZXIgc3R1ZGllcw0KDQoyLiBGb3IgQ2hyb21lIGRhdGEgdGhlIGR1cmF0aW9uIHdhcyBub3QgYSBzaWduaWZpY2FudCBmYWN0b3IgaW4gbGlmdCANCg0KIyBBZHZhbmNlZCBFREENCg0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0W2MoInllYXIiLCJxdWFydGVyIiwicmVnaW9uX3YyIiwicGEiLCJ0YWN0aWMiLCJTaWduaWZpY2FudF9TcGVuZCIsICJjaGFubmVsIildICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSJTZWFyY2giKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAiQWxsIikgJT4lDQogIHBsb3RfYmFyKG5jb2wgPTMpDQogIA0KYGBgDQojIyBQZXJjZW50YWdlIG9mIGVhY2ggdmFyaWFibGUgYnkgUEENCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdFtjKCJ5ZWFyIiwicXVhcnRlciIsInJlZ2lvbl92MiIsInBhIiwidGFjdGljIiwiU2lnbmlmaWNhbnRfU3BlbmQiLCAiY2hhbm5lbCIpXSAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0iU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICBwbG90X2JhcihieSA9ICJwYSIpDQpgYGANCioqKg0KIyNGcmVxdWVuY3kgYW5kIFBlcmNlbnRhZ2UgZm9yIGVhY2ggdmFyaWFibGUgDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RbYygieWVhciIsInF1YXJ0ZXIiLCJyZWdpb25fdjIiLCJwYSIsInRhY3RpYyIsIlNpZ25pZmljYW50X1NwZW5kIiwgImNoYW5uZWwiKV0gJT4lDQogIGZpbHRlcihjaGFubmVsID09ICJTZWFyY2giKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAiQWxsIikgJT4lDQogIEV4cENhdFZpeigpDQogIA0KYGBgDQoNCioqKg0KIyBWaXN1YWxpemluZyAlIG9mIFNpZ25pZmljYW50IFNwZW5kIGJ5IEdyb3VwZWQgQ29udmVyc2lvbiwgVGFjdGljIGFuZCBSZWdpb24gdXNpbmcgRXhwQ2F0Vml6DQojIyAlIG9mIFNpZ25pZmljYW50IFNwZW5kIGJ5IEdyb3VwZWQgQ29udmVyc2lvbg0KYGBge3J9DQpFeHBDYXRWaXooDQogIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogICAgZmlsdGVyKGNoYW5uZWwgPT0gIlNlYXJjaCIpICU+JQ0KICAgIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICAgIHNlbGVjdChncm91cGVkX2NvbnZlcnNpb24sIFNpZ25pZmljYW50X1NwZW5kKSwNCiAgdGFyZ2V0PSJncm91cGVkX2NvbnZlcnNpb24iKQ0KDQpgYGANCg0KKioqDQojIyAlIFNpZ25pZmljYW50IFNwZW5kIGJ5IFRhY3RpYw0KYGBge3J9DQpFeHBDYXRWaXooDQogIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogICAgZmlsdGVyKGNoYW5uZWwgPT0gIlNlYXJjaCIpICU+JQ0KICAgIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICAgIHNlbGVjdCh0YWN0aWMsIFNpZ25pZmljYW50X1NwZW5kKSwNCiAgdGFyZ2V0PSJ0YWN0aWMiKQ0KYGBgDQoqKioNCiMjICUgU2lnbmlmaWNhbnQgU3BlbmQgYnkgUmVnaW9uDQpgYGB7cn0NCkV4cENhdFZpeigNCiAgRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgICBmaWx0ZXIoY2hhbm5lbCA9PSAiU2VhcmNoIikgJT4lDQogICAgZmlsdGVyKHRhY3RpYyAhPSAiQWxsIikgJT4lDQogICAgc2VsZWN0KHJlZ2lvbl92MiwgU2lnbmlmaWNhbnRfU3BlbmQpLA0KICB0YXJnZXQ9InJlZ2lvbl92MiIpDQpgYGANCioqKg0KIyMgJSBvZiBTaWduaWZpY2FudCBTcGVuZCBieSBQQSB3aXRoIHAtdmFsdWVzIA0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAiU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICBnZ2JhcnN0YXRzKA0KICAgIHggPSBTaWduaWZpY2FudF9TcGVuZCwNCiAgICB5ID0gcGEsDQogICAgbGFiZWwgPSAiYm90aCINCiAgKQ0KYGBgDQoqKioNCiMjICUgb2YgU2lnbmlmaWNhbnQgU3BlbmQgYnkgUmVnaW9uIHdpdGggcC12YWx1ZXMNCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0gIlNlYXJjaCIpICU+JQ0KICBmaWx0ZXIodGFjdGljICE9ICJBbGwiKSAlPiUNCiAgZ2diYXJzdGF0cygNCiAgICB4ID0gU2lnbmlmaWNhbnRfU3BlbmQsDQogICAgeSA9IHJlZ2lvbl92MiwNCiAgICBsYWJlbCA9ICJib3RoIg0KICApDQpgYGANCioqKg0KIyMgU3RhdHMgdGFibGUgb2YgYWxsIHZhcmlhYmxlcyBpbiBTZWFyY2gNCmBgYHtyfQ0KZGxvb2tyOjpkZXNjcmliZShGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICAgICAgICAgICAgICAgICAgIGZpbHRlcihjaGFubmVsID09ICJTZWFyY2giKSAlPiUNCiAgICAgICAgICAgICAgICBmaWx0ZXIodGFjdGljICE9ICJBbGwiKSkgJT4lDQogIGZsZXh0YWJsZSgpDQpgYGANCioqKg0KIyMgU3RhdHMgdGFibGUgb2YgYWxsIHZhcmlhYmxlcyBpbiBTZWFyY2ggYnkgcGEgDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09ICJTZWFyY2giKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAiQWxsIikgJT4lDQogIGdyb3VwX2J5KHBhKSAlPiUNCiAgc2VsZWN0KHRyZWF0bWVudF91c2VyX2NvdW50LCBleHBvc2VkLCBjb250cm9sX3VzZXJfY291bnQsIHNjYWxlZF9jb250cm9sLCBjb250cm9sLCBzY2FsaW5nX2ZhY3RvciwNCiAgICAgICAgIGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCwgYWJzb2x1dGVfbGlmdCwgcmVsYXRpdmVfbGlmdCwgZHVyYXRpb24sIHBhKSAlPiUNCiAgdW5pdmFyX251bWVyaWMoKSAlPiUNCiAga25pdHI6OmthYmxlKCkNCmBgYA0KKioqDQojIyBTdGF0cyB0YWJsZSBvZiBhbGwgdmFyaWFibGVzIGluIFNlYXJjaCBkYXRhLCBpbmNsdWRlcyBtb3JlIHRoYW4gYWJvdmUgdGFibGVzIGxpa2UgbWluLCBtYXggYW5kICMgb2Ygb3V0bGllcnMgDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09ICJTZWFyY2giKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAiQWxsIikgJT4lDQogIGdyb3VwX2J5KHBhKSAlPiUNCiAgc2VsZWN0KHRyZWF0bWVudF91c2VyX2NvdW50LCBleHBvc2VkLCBjb250cm9sX3VzZXJfY291bnQsIHNjYWxlZF9jb250cm9sLCBjb250cm9sLCBzY2FsaW5nX2ZhY3RvciwNCiAgICAgICAgIGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCwgYWJzb2x1dGVfbGlmdCwgcmVsYXRpdmVfbGlmdCwgZHVyYXRpb24sIHBhKSAlPiUNCiAgZGlhZ25vc2VfbnVtZXJpYygpICU+JQ0KICBmbGV4dGFibGUoKQ0KYGBgDQoqKioNCg0KIyNTdW1tYXJ5IG9mIEZpbmRpbmdzDQoxLiBQaXhlbCBQQSBoYWQgbG93ZXN0IGF2ZXJhZ2UgbGlmdCBidXQgaGlnaGVzdCBhdmVyYWdlIGR1cmF0aW9uDQoNCjIuIENocm9tZSBQQSBoYWQgaGlnaGVzdCBhdmVyYWdlIGFuZCBtZWRpYW4gbGlmdCBidXQgbG93ZXN0IGF2ZXJhZ2UgIGFuZCBtZWRpYW4gY29zdCANCiMjIEhpc3RvZ3JhbSBvZiBudW1lcmljIGZlYXR1cmVzIGluIFNlYXJjaCBkYXRhIA0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JSANCiAgZmlsdGVyKGNoYW5uZWwgPT0gIlNlYXJjaCIpICU+JQ0KICBmaWx0ZXIodGFjdGljICE9ICJBbGwiKSAlPiUNCiAgc2VsZWN0KGFic29sdXRlX2xpZnQsIGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCwgZHVyYXRpb24sIHJlbGF0aXZlX2xpZnQpICU+JQ0KICBwbG90X2hpc3RvZ3JhbShuY29sID0yKQ0KYGBgDQoNCioqKg0KIyMgUXVhbnRpbGUtUXVhbnRpbGUgcGxvdHMgcGVyIHZhcmlhYmxlIHRvIGNoZWNrIGZvciBub3JtYWxpdHkgaW4gb3JkZXIgdG8gZGV0ZXJtaW5lIGFwcHJvcHJpYXRlIHN0YXRpc3RpY2FsIHRlc3RzDQojIyMgVGhlIGNsb3NlciB0byB0aGUgZGlhZ29uYWwgbGluZSB0aGUgY2xvc2VyIHRvIG5vcm1hbCBkaXN0cmlidXRpb24NCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAiU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICBzZWxlY3QoYWJzb2x1dGVfbGlmdCwgY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCByZWxhdGl2ZV9saWZ0LCBkdXJhdGlvbikgJT4lDQogIHBsb3RfcXEoKQ0KYGBgDQoqKioNCiMjIFF1YW50aWxlLVF1YW50aWxlIHBsb3QgYnkgUEENCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAiU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICBzZWxlY3QoYWJzb2x1dGVfbGlmdCwgY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCByZWxhdGl2ZV9saWZ0LCBkdXJhdGlvbiwgcGEsIGdyb3VwZWRfY29udmVyc2lvbiwgcmVnaW9uX3YyKSAlPiUNCiAgcGxvdF9xcShieSA9ICJwYSIpDQpgYGANCg0KKioqDQojIyBOb3JtYWxpdHkgRGlhZ25vc2lzIFBsb3RzIG9mIEFic29sdXRlIExpZnQgYnkgUEENCmBgYHtyfQ0KIyMgRGlhZ25vc2lzIHBsb3RzIGxldCB5b3Ugc2VlIGhvdyBkaXN0cmlidXRpb25zIHdvdWxkIGNoYW5nZSB3aXRoIGxvZyBhbmQgc3FydCB0cmFuc2Zvcm1hdGlvbiBpbiBjYXNlIG5vcm1hbGl0eSBpc24ndCBtZXQNCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09ICJTZWFyY2giKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAiQWxsIikgJT4lDQogIGdyb3VwX2J5KHBhKSAlPiUNCiAgcGxvdF9ub3JtYWxpdHkoYWJzb2x1dGVfbGlmdCkNCmBgYA0KDQojIyBOb3JtYWxpdHkgRGlhZ25vc2lzIFBsb3RzIG9mIENvc3QgU3BlbnQgYnkgUEENCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0gIlNlYXJjaCIpICU+JQ0KICBmaWx0ZXIodGFjdGljICE9ICJBbGwiKSAlPiUNCiAgZ3JvdXBfYnkocGEpICU+JQ0KICBwbG90X25vcm1hbGl0eShjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXApDQpgYGANCg0KDQoqKioNCiMjIEJveHBsb3RzIG9mIGFic29sdXRlIGxpZnQsIGNvc3Qgc3BlbnQsIGFuZCByZWxhdGl2ZSBsaWZ0IGFjcm9zcyBhbGwgcGEncw0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAiU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICBzZWxlY3QoYWJzb2x1dGVfbGlmdCwgY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCByZWxhdGl2ZV9saWZ0LCBwYSwgcmVnaW9uX3YyLCBncm91cGVkX2NvbnZlcnNpb24pICU+JQ0KcGxvdF9ib3hwbG90KGJ5ID0gInBhIikNCmBgYA0KKioqDQoNCiMjIERpc3RyaWJ1dGlvbnMgb2YgYWJzb2x1dGUgbGlmdCBmb3IgZWFjaCBwYSwgaW5jbHVkaW5nIGNvbXBhcmlzb24gYmV0d2VlbiBwYSBkaXN0cmlidXRpb25zIGFuZCBwLXZhbHVlcw0KYGBge3J9DQojaW5zdGFsbC5wYWNrYWdlcygiUE1DTVJwbHVzIikNCg0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0gICJTZWFyY2giKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAiQWxsIikgJT4lDQogIHNlbGVjdChhYnNvbHV0ZV9saWZ0LCBjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXAsIHBhLCByZWdpb25fdjIsIGdyb3VwZWRfY29udmVyc2lvbikgJT4lDQogIGdnYmV0d2VlbnN0YXRzKHggPSBwYSwgeSA9IGFic29sdXRlX2xpZnQsIHR5cGUgPSAibnAiKQ0KYGBgDQoqKioNCkZpbmRpbmdzOg0KMS4gQWxsIGRpZmZlcmVuY2VzIGluIGRpc3RyaWJ1dGlvbnMgb2YgYWJzb2x1dGUgbGlmdCBiZXR3ZWVuIHBhJ3MgYXJlIHNpZ25pZmljYW50IA0KDQojIyBEaXN0cmlidXRpb25zIG9mIGFic29sdXRlIGxpZnQgZm9yIGVhY2ggcGEsIGV4Y2x1ZGluZyBDaHJvbWUNCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0gICJTZWFyY2giKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAiQWxsIikgJT4lDQogIGZpbHRlcihwYSAhPSAiQ2hyb21lIikgJT4lDQogIHNlbGVjdChhYnNvbHV0ZV9saWZ0LCBjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXAsIHBhLCByZWdpb25fdjIsIGdyb3VwZWRfY29udmVyc2lvbikgJT4lDQogIGdnYmV0d2VlbnN0YXRzKHggPSBwYSwgeSA9IGFic29sdXRlX2xpZnQsIHR5cGUgPSAibnAiKQ0KYGBgDQojIyBEaXN0cmlidXRpb25zIG9mIGFic29sdXRlIGxpZnQgZm9yIGVhY2ggcGEsIGV4Y2x1ZGluZyBDaHJvbWUgYW5kIEdvb2dsZSBDbG91ZA0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAgIlNlYXJjaCIpICU+JQ0KICBmaWx0ZXIodGFjdGljICE9ICJBbGwiKSAlPiUNCiAgZmlsdGVyKCFwYSAlaW4lIGMoIkNocm9tZSIsICJHb29nbGUgQ2xvdWQiKSkgJT4lDQogIHNlbGVjdChhYnNvbHV0ZV9saWZ0LCBjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXAsIHBhLCByZWdpb25fdjIsIGdyb3VwZWRfY29udmVyc2lvbikgJT4lDQogIGdnYmV0d2VlbnN0YXRzKHggPSBwYSwgeSA9IGFic29sdXRlX2xpZnQsIHR5cGUgPSAibnAiKQ0KYGBgDQoNCiMjIERpc3RyaWJ1dGlvbnMgb2YgYWJzb2x1dGUgbGlmdCBmb3IgZWFjaCByZWdpb24sIGluY2x1ZGluZyBjb21wYXJpc29uIGJldHdlZW4gcmVnaW9uIGRpc3RyaWJ1dGlvbnMgYW5kIHAtdmFsdWVzDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09ICAiU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICBzZWxlY3QoYWJzb2x1dGVfbGlmdCwgY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCBwYSwgcmVnaW9uX3YyLCBncm91cGVkX2NvbnZlcnNpb24pICU+JQ0KICBnZ2JldHdlZW5zdGF0cyh4ID0gcmVnaW9uX3YyLCB5ID0gYWJzb2x1dGVfbGlmdCwgdHlwZSA9ICJucCIpDQpgYGANCioqKg0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAgIlNlYXJjaCIpICU+JQ0KICBmaWx0ZXIodGFjdGljICE9ICJBbGwiKSAlPiUNCiAgc2VsZWN0KGFic29sdXRlX2xpZnQsIGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCwgcGEsIHJlZ2lvbl92MiwgZ3JvdXBlZF9jb252ZXJzaW9uKSAlPiUNCiAgZ2diZXR3ZWVuc3RhdHMoeCA9IHBhLCB5ID0gY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCB0eXBlID0gIm5wIikNCmBgYA0KKioqDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09ICAiU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICBmaWx0ZXIocGEgIT0gIkRTTSIpICU+JQ0KICBzZWxlY3QoYWJzb2x1dGVfbGlmdCwgY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCBwYSwgcmVnaW9uX3YyLCBncm91cGVkX2NvbnZlcnNpb24pICU+JQ0KICBnZ2JldHdlZW5zdGF0cyh4ID0gcGEsIHkgPSBjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXAsIHR5cGUgPSAibnAiKQ0KYGBgDQoqKioNCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0gIlNlYXJjaCIpICU+JQ0KICBmaWx0ZXIodGFjdGljICE9ICJBbGwiKSAlPiUNCiAgY29ycmVsYXRlKGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCwgYWJzb2x1dGVfbGlmdCkNCmBgYA0KKioqDQpgYGB7cn0NCmNvcnJlbGF0ZV9mdW5jIDwtIGZ1bmN0aW9uKHBhX3ZhbCkgew0KICBwbG90X2NvcnI8LSBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAiU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICBmaWx0ZXIocGEgPT0gcGFfdmFsKSAlPiUNCiAgY29ycmVsYXRlKGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCwgYWJzb2x1dGVfbGlmdCkNCiAgcHJpbnQocGxvdF9jb3JyKQ0KfQ0KDQpwYV92YWxzID0gYygiQ2hyb21lIiwgIkNocm9tZWJvb2siLCAiRFNNIiwgIlBpeGVsIiwgIkdvb2dsZSBDbG91ZCIpDQoNCmZvciAoaSBpbiBwYV92YWxzKSB7DQogIGNvcnJlbGF0ZV9mdW5jKGkpDQp9DQpgYGANCkZpbmRpbmdzDQoxLiBBbGwgcGEncyBleGNlcHQgUGl4ZWwgaGF2ZSBjb3JyZWxhdGlvbiBjb2VmZmljaWVudHMgPjAuNSBmb3IgYWJzb2x1dGUgbGlmdCBhbmQgY29zdCBzcGVudA0KMi4gQ2hyb21lYm9vayBhbmQgRFNNIGhhdmUgb3ZlciAuOTAgY29ycmVsYXRpb24gYmV0d2VlbiBsaWZ0IGFuZCBjb3N0DQozLiBQaXhlbCBkYXRhIHNob3dzIG5lZ2F0aXZlIGNvcnJlbGF0aW9uIGJldHdlZW4gbGlmdCBhbmQgY29zdCAtIG9ubHkgaGFzIDcgZGF0YXBvaW50cw0KKioqDQojI1RTTkUgZ3JhcGggYnkgZ3JvdXBlZCBjb252ZXJzaW9uDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfMiA8LSANCiAgRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0iU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICBzZWxlY3QoLWMoc3R1ZHlfaWQsIHllYXIsIHF1YXJ0ZXIpKSAlPiUNCiAgbXV0YXRlKElEID0gcm93X251bWJlcigpKQ0KDQojZXh0cmFjdGluZyBjYXRlZ29yaWNhbCBjb2x1bW5zIA0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF9tZXRhIDwtIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfMiAlPiUNCiAgc2VsZWN0KElELCBwYSwgdGFjdGljLCBncm91cGVkX2NvbnZlcnNpb24sIHJlZ2lvbl92MikNCg0KI3RTTkVfZml0IDwtIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfMiAlPiUNCiAjIHNlbGVjdChJRCx0cmVhdG1lbnRfdXNlcl9jb3VudCwgZXhwb3NlZCwgY29udHJvbF91c2VyX2NvdW50LCBjb250cm9sLCBzY2FsaW5nX2ZhY3RvciwNCiAgIyAgICAgICBjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXAsIGFic29sdXRlX2xpZnQsIHJlbGF0aXZlX2xpZnQsIFNpZ25pZmljYW50X1NwZW5kKSAlPiUNCiAgI2NvbHVtbl90b19yb3duYW1lcygiSUQiKSAlPiUNCiAgI3NjYWxlKCkgJT4lDQogICNSdHNuZSgpDQoNCiNTZWxlY3Rpbmcgb25seSBudW1lcmljYWwgY29sdW1ucyBhbmQgZml0dGluZyB0aGUgZGF0YSANCnRTTkVfZml0IDwtIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfMiAlPiUNCiAgc2VsZWN0KElELHRyZWF0bWVudF91c2VyX2NvdW50LCBleHBvc2VkLCBjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXAsIGFic29sdXRlX2xpZnQsIHJlbGF0aXZlX2xpZnQsIFNpZ25pZmljYW50X1NwZW5kKSAlPiUNCiAgY29sdW1uX3RvX3Jvd25hbWVzKCJJRCIpICU+JQ0KICBzY2FsZSgpICU+JQ0KICBSdHNuZSgpDQoNCiN0dXJuaW5nIHJlc3VsdHMgaW50byBkYXRhZnJhbWUgDQp0U05FX2RmIDwtIHRTTkVfZml0JFkgJT4lDQogIGFzLmRhdGEuZnJhbWUoKSAlPiUNCiAgcmVuYW1lKHRTTkUxPSJWMSIsIHRTTkUyPSJWMiIpICU+JQ0KICBtdXRhdGUoSUQgPSByb3dfbnVtYmVyKCkpDQoNCiNqb2luaW5nIGNhdGVnb3JpY2FsIGFuZCBudW1lcmljYWwgY29sdW1ucw0KdFNORV9kZiA8LSB0U05FX2RmICU+JQ0KICBpbm5lcl9qb2luKEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfbWV0YSwgYnkgPSAiSUQiKQ0KDQp0U05FX2RmICU+JSBoZWFkKCkNCg0KdFNORV9kZiAlPiUNCiAgZ2dwbG90KGFlcyh4ID0gdFNORTEsDQogICAgICAgICAgICAgeSA9IHRTTkUyLA0KICAgICAgICAgICAgIGNvbG9yID0gZ3JvdXBlZF9jb252ZXJzaW9uKSkgKw0KICBnZW9tX3BvaW50KCkgKw0KICB0aGVtZShsZWdlbmQucG9zaXRpb249ImJvdHRvbSIpDQogIA0KYGBgDQojIyB0U05FIEdyYXBoIGJ5IHJlZ2lvbiANCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdF8yIDwtIA0KICBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSJTZWFyY2giKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAiQWxsIikgJT4lDQogIHNlbGVjdCgtYyhzdHVkeV9pZCwgeWVhciwgcXVhcnRlcikpICU+JQ0KICBtdXRhdGUoSUQgPSByb3dfbnVtYmVyKCkpDQoNCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3RfbWV0YSA8LSBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0XzIgJT4lDQogIHNlbGVjdChJRCwgcGEsIHRhY3RpYywgZ3JvdXBlZF9jb252ZXJzaW9uLCByZWdpb25fdjIpDQoNCiN0U05FX2ZpdCA8LSBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0XzIgJT4lDQogIyBzZWxlY3QoSUQsdHJlYXRtZW50X3VzZXJfY291bnQsIGV4cG9zZWQsIGNvbnRyb2xfdXNlcl9jb3VudCwgY29udHJvbCwgc2NhbGluZ19mYWN0b3IsDQogICMgICAgICAgY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCBhYnNvbHV0ZV9saWZ0LCByZWxhdGl2ZV9saWZ0LCBTaWduaWZpY2FudF9TcGVuZCkgJT4lDQogICNjb2x1bW5fdG9fcm93bmFtZXMoIklEIikgJT4lDQogICNzY2FsZSgpICU+JQ0KICAjUnRzbmUoKQ0KDQp0U05FX2ZpdCA8LSBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0XzIgJT4lDQogIHNlbGVjdChJRCx0cmVhdG1lbnRfdXNlcl9jb3VudCwgZXhwb3NlZCwgY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCBhYnNvbHV0ZV9saWZ0LCByZWxhdGl2ZV9saWZ0LCBTaWduaWZpY2FudF9TcGVuZCkgJT4lDQogIGNvbHVtbl90b19yb3duYW1lcygiSUQiKSAlPiUNCiAgc2NhbGUoKSAlPiUNCiAgUnRzbmUoKQ0KDQp0U05FX2RmIDwtIHRTTkVfZml0JFkgJT4lDQogIGFzLmRhdGEuZnJhbWUoKSAlPiUNCiAgcmVuYW1lKHRTTkUxPSJWMSIsIHRTTkUyPSJWMiIpICU+JQ0KICBtdXRhdGUoSUQgPSByb3dfbnVtYmVyKCkpDQoNCnRTTkVfZGYgPC0gdFNORV9kZiAlPiUNCiAgaW5uZXJfam9pbihGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0X21ldGEsIGJ5ID0gIklEIikNCg0KdFNORV9kZiAlPiUgaGVhZCgpDQoNCnRTTkVfZGYgJT4lDQogIGdncGxvdChhZXMoeCA9IHRTTkUxLA0KICAgICAgICAgICAgIHkgPSB0U05FMiwNCiAgICAgICAgICAgICBjb2xvciA9IHBhKSkgKw0KICBnZW9tX3BvaW50KCkgKw0KICB0aGVtZShsZWdlbmQucG9zaXRpb249ImJvdHRvbSIpDQpgYGANCg0KIyNTdGF0cyB0YWJsZSBvZiBmZWF0dXJlcyB3aXQgYW5kIHdpdG91dCBPdXRsaWVycyAtIGNhbGN1bGF0ZWQgdXNpbmcgSVFSIG1ldGhvZA0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAiU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICBkaWFnbm9zZV9vdXRsaWVyKCkgJT4lDQogIGZsZXh0YWJsZSgpDQpgYGANCioqKiANCiMjUGxvdHRpbmcgVmFyaWFibGVzIHdpdGggYW5kIHdpdGhvdXQgT3V0bGllcnMNCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0gIlNlYXJjaCIpICU+JQ0KICBmaWx0ZXIodGFjdGljICE9ICJBbGwiKSAlPiUNCiAgZHBseXI6OnNlbGVjdChhYnNvbHV0ZV9saWZ0LCBjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXApICU+JQ0KICBwbG90X291dGxpZXIoKQ0KICANCmBgYA0KDQoqKiogDQojIyBQbG90dGluZyB2YXJpYWJsZXMgd2l0aCBhbmQgd2l0aG91dCBvdXRsaWVycyAtIGV4Y2x1ZGluZyBjaHJvbWUgZGF0YSANCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0gIlNlYXJjaCIpICU+JQ0KICBmaWx0ZXIodGFjdGljICE9ICJBbGwiKSAlPiUNCiAgZmlsdGVyKHBhICE9ICJDaHJvbWUiKSAlPiUNCiAgZHBseXI6OnNlbGVjdChhYnNvbHV0ZV9saWZ0LCBjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXApICU+JQ0KICBwbG90X291dGxpZXIoKQ0KYGBgDQoqKioNCiNQbG90dGluZyB2YXJpYWJsZXMgd2l0aCBhbmQgd2l0aG91dCBvdXRsaWVycyAtIENocm9tZSBkYXRhIA0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAiU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICBmaWx0ZXIocGEgPT0gIkNocm9tZSIpICU+JQ0KICBkcGx5cjo6c2VsZWN0KGFic29sdXRlX2xpZnQsIGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCkgJT4lDQogIHBsb3Rfb3V0bGllcigpDQpgYGANCg0KKioqDQojIyBDaHJvbWUgZGF0YSANCkNocm9tZSBzZWVtcyB0byBiZSBvdXRwZXJmb3JtaW5nIGFsbCBvdGhlciBwYSdzDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09ICJTZWFyY2giKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAiQWxsIikNCmBgYA0KDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09ICJTZWFyY2giKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyA9PSAiQWxsIikgJT4lDQogIGZpbHRlcihhYnNvbHV0ZV9saWZ0ID09IG1heChhYnNvbHV0ZV9saWZ0KSkgJT4lDQogIHNlbGVjdChzdHVkeV9pZCwgcGEpDQpgYGANCg0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAiU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgPT0gIkFsbCIpICU+JQ0KICBmaWx0ZXIoY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwID09IG1heChjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXApKSAlPiUNCiAgc2VsZWN0KHN0dWR5X2lkLCBwYSkNCmBgYA0KIyMgQ2hyb21lIGRhdGEgDQoNCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0gIlNlYXJjaCIpICU+JQ0KICBmaWx0ZXIodGFjdGljICE9ICJBbGwiKSAlPiUNCiAgZmlsdGVyKHBhID09ICJDaHJvbWUiKSAlPiUNCiAgc2VsZWN0KGFic29sdXRlX2xpZnQsIGNvc3Rfc3BlbnRfb25fZXhwb3NlZF9ncm91cCwgcmVsYXRpdmVfbGlmdCwgcGEsIHJlZ2lvbl92MiwgZ3JvdXBlZF9jb252ZXJzaW9uKSAlPiUNCnBsb3RfYm94cGxvdChieSA9ICJyZWdpb25fdjIiKQ0KYGBgDQoqKioNClNpbmNlIENocm9tZSBvbmx5IGNvbnRhaW5zIERlc2t0b3AgRG93bmxvYWRzLCB0aGlzIGNvdWxkIGJlIGNvbnRyaWJ1dGluZyB0byB3aHkgaXQgaGFzIHN1Y2ggaGlnaCBsaWZ0IA0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAiU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICBmaWx0ZXIocGEgPT0gIkNocm9tZSIpICU+JQ0KICBzZWxlY3QoYWJzb2x1dGVfbGlmdCwgY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCByZWxhdGl2ZV9saWZ0LCBwYSwgcmVnaW9uX3YyLCBncm91cGVkX2NvbnZlcnNpb24pICU+JQ0KcGxvdF9ib3hwbG90KGJ5ID0gImdyb3VwZWRfY29udmVyc2lvbiIpDQpgYGANCioqKg0KI3NlZWluZyB3aGF0IHJlZ2lvbnMgYXJlIGluIENocm9tZSANCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0gIlNlYXJjaCIpICU+JQ0KICBmaWx0ZXIodGFjdGljICE9ICJBbGwiKSAlPiUNCiAgZmlsdGVyKHBhID09ICJDaHJvbWUiKSAlPiUNCiAgZ2dwbG90KGFlcyh4ID0gY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCB5ID0gYWJzb2x1dGVfbGlmdCwgY29sb3IgPSByZWdpb25fdjIpKSArDQogICAgZ2VvbV9wb2ludCgpICsNCiAgICAgICB4bGFiKCJDb3N0IikgKw0KICAgICAgIHlsYWIoIkxpZnQiKSArDQogICAgICAgZ2d0aXRsZSgiQ29zdCB2cy4gTGlmdCIpDQogIA0KYGBgDQoqKioNCmBgYHtyfQ0KRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0gIlNlYXJjaCIpICU+JQ0KICBmaWx0ZXIodGFjdGljICE9ICJBbGwiKSAlPiUNCiAgZmlsdGVyKHBhID09ICJDaHJvbWUiKSAlPiUNCiAgZ3JvdXBfYnkocmVnaW9uX3YyKSAlPiUNCiAgc3VtbWFyaXNlKGNvdW50X3JlZ2lvbnMgPSBuKCkpDQpgYGANCg0KKioqDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09ICJTZWFyY2giKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAiQWxsIikgJT4lDQogIGZpbHRlcihwYSA9PSAiQ2hyb21lIikgJT4lDQogIGdyb3VwX2J5KFNpZ25pZmljYW50X1NwZW5kKSAlPiUNCiAgc3VtbWFyaXNlKHRvdGFsX2NvdW50ID0gbigpKQ0KYGBgDQpgYGB7cn0NCnNjYXR0ZXJfc3RhdHNfcGEgPC0gZnVuY3Rpb24ocGFfdmFsKSB7DQogIHBsb3QgPC0gRmluYWxfQ0xTXzIwMjJfU3R1ZHlfTGlzdCAlPiUNCiAgZmlsdGVyKGNoYW5uZWwgPT0gIlNlYXJjaCIpICU+JQ0KICBmaWx0ZXIodGFjdGljICE9ICJBbGwiKSAlPiUNCiAgZmlsdGVyKHBhID09IHBhX3ZhbCkgJT4lDQogIGdnc2NhdHRlcnN0YXRzKA0KICAgIHggPSBjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXAsDQogICAgeSA9IGFic29sdXRlX2xpZnQsIA0KICAgIHR5cGUgPSAibnAiKQ0KICBwcmludChwbG90KQ0KfQ0KZm9yIChpIGluIHBhX3ZhbHMpIHsNCiAgc2NhdHRlcl9zdGF0c19wYShpKQ0KfQ0KYGBgDQoNCmBgYHtyfQ0Kc2NhdHRlcl9zdGF0c19yZWdpb24gPC0gZnVuY3Rpb24ocmVnaW9uX3ZhbCkgew0KICBwbG90IDwtIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09ICJTZWFyY2giKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAiQWxsIikgJT4lDQogIGZpbHRlcihyZWdpb25fdjIgPT0gcmVnaW9uX3ZhbCkgJT4lDQogIGdnc2NhdHRlcnN0YXRzKA0KICAgIHggPSBjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXAsDQogICAgeSA9IGFic29sdXRlX2xpZnQsIA0KICAgIHR5cGUgPSAibnAiKQ0KICBwcmludChwbG90KQ0KfQ0KZm9yIChpIGluIHJlZ2lvbl9jb2xzKSB7DQogIHNjYXR0ZXJfc3RhdHNfcmVnaW9uKGkpDQp9DQpgYGANCiNFdmFsdWF0aW5nIHN0YXRzIHdpdGggYW5kIHdpdGhvdXQgT3V0bGllcnMgZm9yIGFsbCBQQSdzIA0KDQojIyMgQ2hyb21lIA0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAiU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICBmaWx0ZXIocGEgPT0gIkNocm9tZSIpICU+JQ0KICBzZWxlY3QodHJlYXRtZW50X3VzZXJfY291bnQsIGV4cG9zZWQsIGNvbnRyb2xfdXNlcl9jb3VudCwgY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCBhYnNvbHV0ZV9saWZ0KSAlPiUNCiAgZGlhZ25vc2Vfb3V0bGllcigpICU+JQ0KICBmbGV4dGFibGUoKQ0KYGBgDQoNCiMjIyBDaHJvbWVib29rDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09ICJTZWFyY2giKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAiQWxsIikgJT4lDQogIGZpbHRlcihwYSA9PSAiQ2hyb21lYm9vayIpICU+JQ0KICBzZWxlY3QodHJlYXRtZW50X3VzZXJfY291bnQsIGV4cG9zZWQsIGNvbnRyb2xfdXNlcl9jb3VudCwgY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCBhYnNvbHV0ZV9saWZ0KSAlPiUNCiAgZGlhZ25vc2Vfb3V0bGllcigpICU+JQ0KICBmbGV4dGFibGUoKQ0KYGBgDQoNCg0KIyMjIERTTQ0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAiU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICBmaWx0ZXIocGEgPT0gIlBpeGVsIikgJT4lDQogIHNlbGVjdCh0cmVhdG1lbnRfdXNlcl9jb3VudCwgZXhwb3NlZCwgY29udHJvbF91c2VyX2NvdW50LCBjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXAsIGFic29sdXRlX2xpZnQpICU+JQ0KICBkaWFnbm9zZV9vdXRsaWVyKCkgJT4lDQogIGZsZXh0YWJsZSgpDQpgYGANCg0KIyMjIFBpeGVsDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09ICJTZWFyY2giKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAiQWxsIikgJT4lDQogIGZpbHRlcihwYSA9PSAiQ2hyb21lIikgJT4lDQogIHNlbGVjdCh0cmVhdG1lbnRfdXNlcl9jb3VudCwgZXhwb3NlZCwgY29udHJvbF91c2VyX2NvdW50LCBjb3N0X3NwZW50X29uX2V4cG9zZWRfZ3JvdXAsIGFic29sdXRlX2xpZnQpICU+JQ0KICBkaWFnbm9zZV9vdXRsaWVyKCkgJT4lDQogIGZsZXh0YWJsZSgpDQpgYGANCg0KIyMjIEdvb2dsZSBDbG91ZA0KYGBge3J9DQpGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAiU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICBmaWx0ZXIocGEgPT0gIkdvb2dsZSBDbG91ZCIpICU+JQ0KICBzZWxlY3QodHJlYXRtZW50X3VzZXJfY291bnQsIGV4cG9zZWQsIGNvbnRyb2xfdXNlcl9jb3VudCwgY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCBhYnNvbHV0ZV9saWZ0KSAlPiUNCiAgZGlhZ25vc2Vfb3V0bGllcigpICU+JQ0KICBmbGV4dGFibGUoKQ0KYGBgDQpgYGB7cn0NCiNpbnN0YWxsLnBhY2thZ2VzKCJndHN1bW1hcnkiKQ0KYGBgDQpgYGB7cn0NCkZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09ICJTZWFyY2giKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAiQWxsIikgJT4lDQogIHNlbGVjdChyZWdpb25fdjIsIHBhLCBjaGFubmVsLCB0YWN0aWMsIGdyb3VwZWRfY29udmVyc2lvbiwgY29zdF9zcGVudF9vbl9leHBvc2VkX2dyb3VwLCBhYnNvbHV0ZV9saWZ0KSAlPiUNCiAgdGJsX3N1bW1hcnkoYnkgPSBwYSkgJT4lDQogIGFkZF9wKCkNCmBgYA0KDQoqKioNCmBgYHtyfQ0KU2VhcmNoX25vX3RhY3RpYyA8LSBGaW5hbF9DTFNfMjAyMl9TdHVkeV9MaXN0ICU+JQ0KICBmaWx0ZXIoY2hhbm5lbCA9PSAiU2VhcmNoIikgJT4lDQogIGZpbHRlcih0YWN0aWMgIT0gIkFsbCIpICU+JQ0KICBrcnVza2FsLnRlc3QoYWJzb2x1dGVfbGlmdCB+IHBhKQ0KYGBgDQoNCg0KYGBge3J9DQpTZWFyY2hfbm9fdGFjdGljIDwtIEZpbmFsX0NMU18yMDIyX1N0dWR5X0xpc3QgJT4lDQogIGZpbHRlcihjaGFubmVsID09ICJTZWFyY2giKSAlPiUNCiAgZmlsdGVyKHRhY3RpYyAhPSAiQWxsIikNCmBgYA0KDQoNCmBgYHtyfQ0Ka3J1c2thbC50ZXN0KGFic29sdXRlX2xpZnQgfiBwYSwgZGF0YSA9IFNlYXJjaF9ub190YWN0aWMpDQpgYGANCg0KYGBge3J9DQprcnVza2FsLnRlc3QoYWJzb2x1dGVfbGlmdCB+IHJlZ2lvbl92MiwgZGF0YSA9IFNlYXJjaF9ub190YWN0aWMpDQoNCmBgYA0KDQpgYGB7cn0NCnBhaXJ3aXNlLndpbGNveC50ZXN0KFNlYXJjaF9ub190YWN0aWMkYWJzb2x1dGVfbGlmdCwgU2VhcmNoX25vX3RhY3RpYyRwYSwNCiAgICAgICAgICAgICAgICAgcC5hZGp1c3QubWV0aG9kID0gIkJIIikNCmBgYA0KDQo=